Line data Source code
1 : // Copyright (C) 2016 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.notedb;
16 :
17 : import static com.google.common.base.MoreObjects.firstNonNull;
18 : import static com.google.common.base.Preconditions.checkArgument;
19 : import static com.google.common.base.Preconditions.checkNotNull;
20 : import static com.google.common.base.Preconditions.checkState;
21 : import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
22 : import static com.google.gerrit.server.logging.TraceContext.newTimer;
23 :
24 : import com.google.common.collect.ImmutableList;
25 : import com.google.common.collect.ImmutableListMultimap;
26 : import com.google.common.collect.ListMultimap;
27 : import com.google.common.collect.MultimapBuilder;
28 : import com.google.gerrit.common.Nullable;
29 : import com.google.gerrit.entities.AttentionSetUpdate;
30 : import com.google.gerrit.entities.Change;
31 : import com.google.gerrit.entities.Project;
32 : import com.google.gerrit.entities.ProjectChangeKey;
33 : import com.google.gerrit.entities.RefNames;
34 : import com.google.gerrit.exceptions.StorageException;
35 : import com.google.gerrit.git.RefUpdateUtil;
36 : import com.google.gerrit.metrics.Timer0;
37 : import com.google.gerrit.server.GerritPersonIdent;
38 : import com.google.gerrit.server.cancellation.RequestStateContext;
39 : import com.google.gerrit.server.cancellation.RequestStateContext.NonCancellableOperationContext;
40 : import com.google.gerrit.server.config.AllUsersName;
41 : import com.google.gerrit.server.config.GerritServerConfig;
42 : import com.google.gerrit.server.git.GitRepositoryManager;
43 : import com.google.gerrit.server.logging.Metadata;
44 : import com.google.gerrit.server.logging.TraceContext;
45 : import com.google.gerrit.server.update.BatchUpdateListener;
46 : import com.google.gerrit.server.update.ChainedReceiveCommands;
47 : import com.google.inject.Inject;
48 : import com.google.inject.Provider;
49 : import com.google.inject.assistedinject.Assisted;
50 : import java.io.IOException;
51 : import java.util.Collection;
52 : import java.util.HashSet;
53 : import java.util.Map;
54 : import java.util.Optional;
55 : import java.util.Set;
56 : import org.eclipse.jgit.errors.ConfigInvalidException;
57 : import org.eclipse.jgit.lib.BatchRefUpdate;
58 : import org.eclipse.jgit.lib.Config;
59 : import org.eclipse.jgit.lib.ObjectId;
60 : import org.eclipse.jgit.lib.ObjectInserter;
61 : import org.eclipse.jgit.lib.PersonIdent;
62 : import org.eclipse.jgit.lib.Ref;
63 : import org.eclipse.jgit.lib.Repository;
64 : import org.eclipse.jgit.revwalk.RevWalk;
65 : import org.eclipse.jgit.transport.PushCertificate;
66 : import org.eclipse.jgit.transport.ReceiveCommand;
67 :
68 : /**
69 : * Object to manage a single sequence of updates to NoteDb.
70 : *
71 : * <p>Instances are one-time-use. Handles updating both the change repo and the All-Users repo for
72 : * any affected changes, with proper ordering.
73 : *
74 : * <p>To see the state that would be applied prior to executing the full sequence of updates, use
75 : * {@link #stage()}.
76 : */
77 : public class NoteDbUpdateManager implements AutoCloseable {
78 : private static final int MAX_UPDATES_DEFAULT = 1000;
79 : /** Limits the number of patch sets that can be created. Can be overridden in the config. */
80 : private static final int MAX_PATCH_SETS_DEFAULT = 1000;
81 :
82 : public interface Factory {
83 : NoteDbUpdateManager create(Project.NameKey projectName);
84 : }
85 :
86 : private final Provider<PersonIdent> serverIdent;
87 : private final GitRepositoryManager repoManager;
88 : private final AllUsersName allUsersName;
89 : private final NoteDbMetrics metrics;
90 : private final Project.NameKey projectName;
91 : private final int maxUpdates;
92 : private final int maxPatchSets;
93 : private final ListMultimap<String, ChangeUpdate> changeUpdates;
94 : private final ListMultimap<String, ChangeDraftUpdate> draftUpdates;
95 : private final ListMultimap<String, RobotCommentUpdate> robotCommentUpdates;
96 : private final ListMultimap<String, NoteDbRewriter> rewriters;
97 : private final Set<Change.Id> changesToDelete;
98 :
99 : private OpenRepo changeRepo;
100 : private OpenRepo allUsersRepo;
101 : private AllUsersAsyncUpdate updateAllUsersAsync;
102 : private boolean executed;
103 : private String refLogMessage;
104 : private PersonIdent refLogIdent;
105 : private PushCertificate pushCert;
106 : private ImmutableList<BatchUpdateListener> batchUpdateListeners;
107 :
108 : @Inject
109 : NoteDbUpdateManager(
110 : @GerritServerConfig Config cfg,
111 : @GerritPersonIdent Provider<PersonIdent> serverIdent,
112 : GitRepositoryManager repoManager,
113 : AllUsersName allUsersName,
114 : NoteDbMetrics metrics,
115 : AllUsersAsyncUpdate updateAllUsersAsync,
116 110 : @Assisted Project.NameKey projectName) {
117 110 : this.serverIdent = serverIdent;
118 110 : this.repoManager = repoManager;
119 110 : this.allUsersName = allUsersName;
120 110 : this.metrics = metrics;
121 110 : this.updateAllUsersAsync = updateAllUsersAsync;
122 110 : this.projectName = projectName;
123 110 : maxUpdates = cfg.getInt("change", null, "maxUpdates", MAX_UPDATES_DEFAULT);
124 110 : maxPatchSets = cfg.getInt("change", null, "maxPatchSets", MAX_PATCH_SETS_DEFAULT);
125 110 : changeUpdates = MultimapBuilder.hashKeys().arrayListValues().build();
126 110 : draftUpdates = MultimapBuilder.hashKeys().arrayListValues().build();
127 110 : robotCommentUpdates = MultimapBuilder.hashKeys().arrayListValues().build();
128 110 : rewriters = MultimapBuilder.hashKeys().arrayListValues().build();
129 110 : changesToDelete = new HashSet<>();
130 110 : batchUpdateListeners = ImmutableList.of();
131 110 : }
132 :
133 : @Override
134 : public void close() {
135 : try {
136 110 : if (allUsersRepo != null) {
137 32 : OpenRepo r = allUsersRepo;
138 32 : allUsersRepo = null;
139 32 : r.close();
140 : }
141 : } finally {
142 110 : if (changeRepo != null) {
143 110 : OpenRepo r = changeRepo;
144 110 : changeRepo = null;
145 110 : r.close();
146 : }
147 : }
148 110 : }
149 :
150 : public NoteDbUpdateManager setChangeRepo(
151 : Repository repo, RevWalk rw, @Nullable ObjectInserter ins, ChainedReceiveCommands cmds) {
152 110 : checkState(changeRepo == null, "change repo already initialized");
153 110 : changeRepo = new OpenRepo(repo, rw, ins, cmds, false);
154 110 : return this;
155 : }
156 :
157 : public NoteDbUpdateManager setRefLogMessage(String message) {
158 110 : this.refLogMessage = message;
159 110 : return this;
160 : }
161 :
162 : public NoteDbUpdateManager setRefLogIdent(PersonIdent ident) {
163 110 : this.refLogIdent = ident;
164 110 : return this;
165 : }
166 :
167 : /**
168 : * Set a push certificate for the push that originally triggered this NoteDb update.
169 : *
170 : * <p>The pusher will not necessarily have specified any of the NoteDb refs explicitly, such as
171 : * when processing a push to {@code refs/for/master}. That's fine; this is just passed to the
172 : * underlying {@link BatchRefUpdate}, and the implementation decides what to do with it.
173 : *
174 : * <p>The cert should be associated with the main repo. There is currently no way of associating a
175 : * push cert with the {@code All-Users} repo, since it is not currently possible to update draft
176 : * changes via push.
177 : *
178 : * @param pushCert push certificate; may be null.
179 : * @return this
180 : */
181 : public NoteDbUpdateManager setPushCertificate(PushCertificate pushCert) {
182 110 : this.pushCert = pushCert;
183 110 : return this;
184 : }
185 :
186 : public NoteDbUpdateManager setBatchUpdateListeners(
187 : ImmutableList<BatchUpdateListener> batchUpdateListeners) {
188 110 : checkNotNull(batchUpdateListeners);
189 110 : this.batchUpdateListeners = batchUpdateListeners;
190 110 : return this;
191 : }
192 :
193 : public boolean isExecuted() {
194 110 : return executed;
195 : }
196 :
197 : private void initChangeRepo() throws IOException {
198 109 : if (changeRepo == null) {
199 2 : changeRepo = OpenRepo.open(repoManager, projectName);
200 : }
201 109 : }
202 :
203 : private void initAllUsersRepo() throws IOException {
204 32 : if (allUsersRepo == null) {
205 32 : allUsersRepo = OpenRepo.open(repoManager, allUsersName);
206 : }
207 32 : }
208 :
209 : private boolean isEmpty() {
210 110 : return changeUpdates.isEmpty()
211 60 : && draftUpdates.isEmpty()
212 59 : && robotCommentUpdates.isEmpty()
213 59 : && rewriters.isEmpty()
214 59 : && changesToDelete.isEmpty()
215 57 : && !hasCommands(changeRepo)
216 55 : && !hasCommands(allUsersRepo)
217 110 : && updateAllUsersAsync.isEmpty();
218 : }
219 :
220 : private static boolean hasCommands(@Nullable OpenRepo or) {
221 57 : return or != null && !or.cmds.isEmpty();
222 : }
223 :
224 : /**
225 : * Add an update to the list of updates to execute.
226 : *
227 : * <p>Updates should only be added to the manager after all mutations have been made, as this
228 : * method may eagerly access the update.
229 : *
230 : * @param update the update to add.
231 : */
232 : public void add(ChangeUpdate update) {
233 103 : checkNotExecuted();
234 103 : checkArgument(
235 103 : update.getProjectName().equals(projectName),
236 : "update for project %s cannot be added to manager for project %s",
237 103 : update.getProjectName(),
238 : projectName);
239 103 : checkArgument(
240 103 : !rewriters.containsKey(update.getRefName()),
241 : "cannot update & rewrite ref %s in one BatchUpdate",
242 103 : update.getRefName());
243 :
244 103 : ChangeDraftUpdate du = update.getDraftUpdate();
245 103 : if (du != null) {
246 29 : draftUpdates.put(du.getRefName(), du);
247 : }
248 103 : RobotCommentUpdate rcu = update.getRobotCommentUpdate();
249 103 : if (rcu != null) {
250 9 : robotCommentUpdates.put(rcu.getRefName(), rcu);
251 : }
252 103 : DeleteCommentRewriter deleteCommentRewriter = update.getDeleteCommentRewriter();
253 103 : if (deleteCommentRewriter != null) {
254 : // Checks whether there is any ChangeUpdate or rewriter added earlier for the same ref.
255 3 : checkArgument(
256 3 : !changeUpdates.containsKey(deleteCommentRewriter.getRefName()),
257 : "cannot update & rewrite ref %s in one BatchUpdate",
258 3 : deleteCommentRewriter.getRefName());
259 3 : checkArgument(
260 3 : !rewriters.containsKey(deleteCommentRewriter.getRefName()),
261 : "cannot rewrite the same ref %s in one BatchUpdate",
262 3 : deleteCommentRewriter.getRefName());
263 3 : rewriters.put(deleteCommentRewriter.getRefName(), deleteCommentRewriter);
264 : }
265 :
266 103 : DeleteChangeMessageRewriter deleteChangeMessageRewriter =
267 103 : update.getDeleteChangeMessageRewriter();
268 103 : if (deleteChangeMessageRewriter != null) {
269 : // Checks whether there is any ChangeUpdate or rewriter added earlier for the same ref.
270 1 : checkArgument(
271 1 : !changeUpdates.containsKey(deleteChangeMessageRewriter.getRefName()),
272 : "cannot update & rewrite ref %s in one BatchUpdate",
273 1 : deleteChangeMessageRewriter.getRefName());
274 1 : checkArgument(
275 1 : !rewriters.containsKey(deleteChangeMessageRewriter.getRefName()),
276 : "cannot rewrite the same ref %s in one BatchUpdate",
277 1 : deleteChangeMessageRewriter.getRefName());
278 1 : rewriters.put(deleteChangeMessageRewriter.getRefName(), deleteChangeMessageRewriter);
279 : }
280 :
281 103 : changeUpdates.put(update.getRefName(), update);
282 103 : }
283 :
284 : public void add(ChangeDraftUpdate draftUpdate) {
285 1 : checkNotExecuted();
286 1 : draftUpdates.put(draftUpdate.getRefName(), draftUpdate);
287 1 : }
288 :
289 : public void deleteChange(Change.Id id) {
290 11 : checkNotExecuted();
291 11 : changesToDelete.add(id);
292 11 : }
293 :
294 : /**
295 : * Stage updates in the manager's internal list of commands.
296 : *
297 : * @throws IOException if a storage layer error occurs.
298 : */
299 : private void stage() throws IOException {
300 109 : try (Timer0.Context timer = metrics.stageUpdateLatency.start()) {
301 109 : if (isEmpty()) {
302 0 : return;
303 : }
304 :
305 109 : initChangeRepo();
306 109 : if (!draftUpdates.isEmpty() || !changesToDelete.isEmpty()) {
307 32 : initAllUsersRepo();
308 : }
309 109 : addCommands();
310 0 : }
311 109 : }
312 :
313 : @Nullable
314 : public BatchRefUpdate execute() throws IOException {
315 2 : return execute(false);
316 : }
317 :
318 : @Nullable
319 : public BatchRefUpdate execute(boolean dryrun) throws IOException {
320 110 : checkNotExecuted();
321 110 : if (isEmpty()) {
322 55 : executed = true;
323 55 : return null;
324 : }
325 109 : try (Timer0.Context timer = metrics.updateLatency.start();
326 : NonCancellableOperationContext nonCancellableOperationContext =
327 109 : RequestStateContext.startNonCancellableOperation()) {
328 109 : stage();
329 : // ChangeUpdates must execute before ChangeDraftUpdates.
330 : //
331 : // ChangeUpdate will automatically delete draft comments for any published
332 : // comments, but the updates to the two repos don't happen atomically.
333 : // Thus if the change meta update succeeds and the All-Users update fails,
334 : // we may have stale draft comments. Doing it in this order allows stale
335 : // comments to be filtered out by ChangeNotes, reflecting the fact that
336 : // comments can only go from DRAFT to PUBLISHED, not vice versa.
337 : BatchRefUpdate result;
338 109 : try (TraceContext.TraceTimer ignored =
339 109 : newTimer("NoteDbUpdateManager#updateRepo", Metadata.empty())) {
340 109 : result = execute(changeRepo, dryrun, pushCert);
341 : }
342 109 : try (TraceContext.TraceTimer ignored =
343 109 : newTimer("NoteDbUpdateManager#updateAllUsersSync", Metadata.empty())) {
344 109 : execute(allUsersRepo, dryrun, null);
345 : }
346 109 : if (!dryrun) {
347 : // Only execute the asynchronous operation if we are not in dry-run mode: The dry run would
348 : // have to run synchronous to be of any value at all. For the removal of draft comments from
349 : // All-Users we don't care much of the operation succeeds, so we are skipping the dry run
350 : // altogether.
351 109 : updateAllUsersAsync.execute(refLogIdent, refLogMessage, pushCert);
352 : }
353 109 : executed = true;
354 109 : return result;
355 : } finally {
356 109 : close();
357 : }
358 : }
359 :
360 : public ImmutableListMultimap<ProjectChangeKey, AttentionSetUpdate> attentionSetUpdates() {
361 110 : return this.changeUpdates.values().stream()
362 110 : .collect(
363 110 : flatteningToImmutableListMultimap(
364 103 : cu -> ProjectChangeKey.create(cu.getProjectName(), cu.getId()),
365 103 : cu -> cu.getAttentionSetUpdates().stream()));
366 : }
367 :
368 : @Nullable
369 : private BatchRefUpdate execute(OpenRepo or, boolean dryrun, @Nullable PushCertificate pushCert)
370 : throws IOException {
371 109 : if (or == null || or.cmds.isEmpty()) {
372 109 : return null;
373 : }
374 109 : if (!dryrun) {
375 109 : or.flush();
376 : } else {
377 : // OpenRepo buffers objects separately; caller may assume that objects are available in the
378 : // inserter it previously passed via setChangeRepo.
379 0 : or.flushToFinalInserter();
380 : }
381 :
382 109 : BatchRefUpdate bru = or.repo.getRefDatabase().newBatchUpdate();
383 109 : bru.setPushCertificate(pushCert);
384 109 : if (refLogMessage != null) {
385 100 : bru.setRefLogMessage(refLogMessage, false);
386 : } else {
387 91 : bru.setRefLogMessage(
388 91 : firstNonNull(NoteDbUtil.guessRestApiHandler(), "Update NoteDb refs"), false);
389 : }
390 109 : bru.setRefLogIdent(refLogIdent != null ? refLogIdent : serverIdent.get());
391 109 : bru.setAtomic(true);
392 109 : or.cmds.addTo(bru);
393 109 : bru.setAllowNonFastForwards(allowNonFastForwards(or.cmds));
394 109 : for (BatchUpdateListener listener : batchUpdateListeners) {
395 53 : bru = listener.beforeUpdateRefs(bru);
396 53 : }
397 :
398 109 : if (!dryrun) {
399 109 : RefUpdateUtil.executeChecked(bru, or.rw);
400 : }
401 109 : return bru;
402 : }
403 :
404 : private void addCommands() throws IOException {
405 109 : changeRepo.addUpdates(changeUpdates, Optional.of(maxUpdates), Optional.of(maxPatchSets));
406 109 : if (!draftUpdates.isEmpty()) {
407 29 : boolean publishOnly = draftUpdates.values().stream().allMatch(ChangeDraftUpdate::canRunAsync);
408 29 : if (publishOnly) {
409 26 : updateAllUsersAsync.setDraftUpdates(draftUpdates);
410 : } else {
411 21 : allUsersRepo.addUpdatesNoLimits(draftUpdates);
412 : }
413 : }
414 109 : if (!robotCommentUpdates.isEmpty()) {
415 9 : changeRepo.addUpdatesNoLimits(robotCommentUpdates);
416 : }
417 109 : if (!rewriters.isEmpty()) {
418 4 : addRewrites(rewriters, changeRepo);
419 : }
420 :
421 109 : for (Change.Id id : changesToDelete) {
422 11 : doDelete(id);
423 11 : }
424 109 : }
425 :
426 : private void doDelete(Change.Id id) throws IOException {
427 11 : String metaRef = RefNames.changeMetaRef(id);
428 11 : Optional<ObjectId> old = changeRepo.cmds.get(metaRef);
429 11 : old.ifPresent(
430 0 : objectId -> changeRepo.cmds.add(new ReceiveCommand(objectId, ObjectId.zeroId(), metaRef)));
431 :
432 : // Just scan repo for ref names, but get "old" values from cmds.
433 : for (Ref r :
434 11 : allUsersRepo.repo.getRefDatabase().getRefsByPrefix(RefNames.refsDraftCommentsPrefix(id))) {
435 1 : old = allUsersRepo.cmds.get(r.getName());
436 1 : old.ifPresent(
437 : objectId ->
438 1 : allUsersRepo.cmds.add(new ReceiveCommand(objectId, ObjectId.zeroId(), r.getName())));
439 1 : }
440 11 : }
441 :
442 : private void checkNotExecuted() {
443 110 : checkState(!executed, "update has already been executed");
444 110 : }
445 :
446 : private static void addRewrites(ListMultimap<String, NoteDbRewriter> rewriters, OpenRepo openRepo)
447 : throws IOException {
448 4 : for (Map.Entry<String, Collection<NoteDbRewriter>> entry : rewriters.asMap().entrySet()) {
449 4 : String refName = entry.getKey();
450 4 : ObjectId oldTip = openRepo.cmds.get(refName).orElse(ObjectId.zeroId());
451 :
452 4 : if (oldTip.equals(ObjectId.zeroId())) {
453 0 : throw new StorageException(String.format("Ref %s is empty", refName));
454 : }
455 :
456 4 : ObjectId currTip = oldTip;
457 : try {
458 4 : for (NoteDbRewriter noteDbRewriter : entry.getValue()) {
459 4 : ObjectId nextTip =
460 4 : noteDbRewriter.rewriteCommitHistory(openRepo.rw, openRepo.tempIns, currTip);
461 4 : if (nextTip != null) {
462 4 : currTip = nextTip;
463 : }
464 4 : }
465 0 : } catch (ConfigInvalidException e) {
466 0 : throw new StorageException("Cannot rewrite commit history", e);
467 4 : }
468 :
469 4 : if (!oldTip.equals(currTip)) {
470 4 : openRepo.cmds.add(new ReceiveCommand(oldTip, currTip, refName));
471 : }
472 4 : }
473 4 : }
474 :
475 : /**
476 : * Returns true if we should allow non-fast-forwards while performing the batch ref update. Non-ff
477 : * updates are necessary in some specific cases:
478 : *
479 : * <p>1. Draft ref updates are non fast-forward, since the ref always points to a single commit
480 : * that has no parents.
481 : *
482 : * <p>2. NoteDb rewriters.
483 : *
484 : * <p>3. If any of the receive commands is of type {@link
485 : * org.eclipse.jgit.transport.ReceiveCommand.Type#UPDATE_NONFASTFORWARD} (for example due to a
486 : * force push).
487 : *
488 : * <p>Note that we don't need to explicitly allow non fast-forward updates for DELETE commands
489 : * since JGit forces the update implicitly in this case.
490 : */
491 : private boolean allowNonFastForwards(ChainedReceiveCommands receiveCommands) {
492 109 : return !draftUpdates.isEmpty()
493 109 : || !rewriters.isEmpty()
494 109 : || receiveCommands.getCommands().values().stream()
495 109 : .anyMatch(cmd -> cmd.getType().equals(ReceiveCommand.Type.UPDATE_NONFASTFORWARD));
496 : }
497 : }
|
