Line data Source code
1 : // Copyright (C) 2008 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.submit;
16 :
17 : import static com.google.common.base.MoreObjects.firstNonNull;
18 : import static com.google.common.base.Preconditions.checkArgument;
19 : import static java.util.Comparator.comparing;
20 : import static java.util.Objects.requireNonNull;
21 : import static java.util.stream.Collectors.toSet;
22 :
23 : import com.github.rholder.retry.Attempt;
24 : import com.github.rholder.retry.RetryListener;
25 : import com.google.auto.value.AutoValue;
26 : import com.google.common.base.Joiner;
27 : import com.google.common.collect.ImmutableList;
28 : import com.google.common.collect.ImmutableMap;
29 : import com.google.common.collect.ImmutableSet;
30 : import com.google.common.collect.ImmutableSetMultimap;
31 : import com.google.common.collect.ListMultimap;
32 : import com.google.common.collect.MultimapBuilder;
33 : import com.google.common.collect.SetMultimap;
34 : import com.google.common.flogger.FluentLogger;
35 : import com.google.gerrit.common.Nullable;
36 : import com.google.gerrit.entities.BranchNameKey;
37 : import com.google.gerrit.entities.Change;
38 : import com.google.gerrit.entities.Change.Status;
39 : import com.google.gerrit.entities.PatchSet;
40 : import com.google.gerrit.entities.Project;
41 : import com.google.gerrit.entities.SubmissionId;
42 : import com.google.gerrit.entities.SubmitRecord;
43 : import com.google.gerrit.entities.SubmitRequirement;
44 : import com.google.gerrit.entities.SubmitRequirementResult;
45 : import com.google.gerrit.entities.SubmitTypeRecord;
46 : import com.google.gerrit.exceptions.MergeUpdateException;
47 : import com.google.gerrit.exceptions.StorageException;
48 : import com.google.gerrit.extensions.api.changes.NotifyHandling;
49 : import com.google.gerrit.extensions.api.changes.SubmitInput;
50 : import com.google.gerrit.extensions.client.SubmitType;
51 : import com.google.gerrit.extensions.restapi.AuthException;
52 : import com.google.gerrit.extensions.restapi.ResourceConflictException;
53 : import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
54 : import com.google.gerrit.extensions.restapi.RestApiException;
55 : import com.google.gerrit.git.LockFailureException;
56 : import com.google.gerrit.metrics.Counter0;
57 : import com.google.gerrit.metrics.Description;
58 : import com.google.gerrit.metrics.MetricMaker;
59 : import com.google.gerrit.server.ChangeMessagesUtil;
60 : import com.google.gerrit.server.ChangeUtil;
61 : import com.google.gerrit.server.IdentifiedUser;
62 : import com.google.gerrit.server.InternalUser;
63 : import com.google.gerrit.server.change.NotifyResolver;
64 : import com.google.gerrit.server.git.CodeReviewCommit;
65 : import com.google.gerrit.server.git.MergeTip;
66 : import com.google.gerrit.server.git.validators.MergeValidationException;
67 : import com.google.gerrit.server.git.validators.MergeValidators;
68 : import com.google.gerrit.server.logging.RequestId;
69 : import com.google.gerrit.server.logging.TraceContext;
70 : import com.google.gerrit.server.notedb.ChangeNotes;
71 : import com.google.gerrit.server.notedb.StoreSubmitRequirementsOp;
72 : import com.google.gerrit.server.permissions.PermissionBackendException;
73 : import com.google.gerrit.server.project.NoSuchProjectException;
74 : import com.google.gerrit.server.project.SubmitRuleOptions;
75 : import com.google.gerrit.server.query.change.ChangeData;
76 : import com.google.gerrit.server.query.change.InternalChangeQuery;
77 : import com.google.gerrit.server.submit.MergeOpRepoManager.OpenBranch;
78 : import com.google.gerrit.server.submit.MergeOpRepoManager.OpenRepo;
79 : import com.google.gerrit.server.update.BatchUpdate;
80 : import com.google.gerrit.server.update.BatchUpdateOp;
81 : import com.google.gerrit.server.update.ChangeContext;
82 : import com.google.gerrit.server.update.RetryHelper;
83 : import com.google.gerrit.server.update.SubmissionExecutor;
84 : import com.google.gerrit.server.update.SubmissionListener;
85 : import com.google.gerrit.server.update.SuperprojectUpdateOnSubmission;
86 : import com.google.gerrit.server.update.UpdateException;
87 : import com.google.gerrit.server.util.time.TimeUtil;
88 : import com.google.inject.Inject;
89 : import com.google.inject.Provider;
90 : import com.google.inject.Singleton;
91 : import java.io.IOException;
92 : import java.time.Instant;
93 : import java.util.ArrayList;
94 : import java.util.Collection;
95 : import java.util.HashMap;
96 : import java.util.HashSet;
97 : import java.util.LinkedHashSet;
98 : import java.util.List;
99 : import java.util.Map;
100 : import java.util.Optional;
101 : import java.util.Set;
102 : import java.util.function.Function;
103 : import java.util.stream.Collectors;
104 : import org.eclipse.jgit.errors.ConfigInvalidException;
105 : import org.eclipse.jgit.errors.IncorrectObjectTypeException;
106 : import org.eclipse.jgit.lib.Constants;
107 : import org.eclipse.jgit.lib.ObjectId;
108 : import org.eclipse.jgit.lib.Ref;
109 : import org.eclipse.jgit.revwalk.RevCommit;
110 :
111 : /**
112 : * Merges changes in submission order into a single branch.
113 : *
114 : * <p>Branches are reduced to the minimum number of heads needed to merge everything. This allows
115 : * commits to be entered into the queue in any order (such as ancestors before descendants) and only
116 : * the most recent commit on any line of development will be merged. All unmerged commits along a
117 : * line of development must be in the submission queue in order to merge the tip of that line.
118 : *
119 : * <p>Conflicts are handled by discarding the entire line of development and marking it as
120 : * conflicting, even if an earlier commit along that same line can be merged cleanly.
121 : */
122 : public class MergeOp implements AutoCloseable {
123 76 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
124 :
125 76 : private static final SubmitRuleOptions SUBMIT_RULE_OPTIONS = SubmitRuleOptions.builder().build();
126 76 : private static final SubmitRuleOptions SUBMIT_RULE_OPTIONS_ALLOW_CLOSED =
127 76 : SUBMIT_RULE_OPTIONS.toBuilder().recomputeOnClosedChanges(true).build();
128 :
129 : public static class CommitStatus {
130 : private final ImmutableMap<Change.Id, ChangeData> changes;
131 : private final ImmutableSetMultimap<BranchNameKey, Change.Id> byBranch;
132 : private final Map<Change.Id, CodeReviewCommit> commits;
133 : private final ListMultimap<Change.Id, String> problems;
134 : private final boolean allowClosed;
135 :
136 53 : private CommitStatus(ChangeSet cs, boolean allowClosed) {
137 53 : checkArgument(
138 53 : !cs.furtherHiddenChanges(), "CommitStatus must not be called with hidden changes");
139 53 : changes = cs.changesById();
140 53 : ImmutableSetMultimap.Builder<BranchNameKey, Change.Id> bb = ImmutableSetMultimap.builder();
141 53 : for (ChangeData cd : cs.changes()) {
142 53 : bb.put(cd.change().getDest(), cd.getId());
143 53 : }
144 53 : byBranch = bb.build();
145 53 : commits = new HashMap<>();
146 53 : problems = MultimapBuilder.treeKeys(comparing(Change.Id::get)).arrayListValues(1).build();
147 53 : this.allowClosed = allowClosed;
148 53 : }
149 :
150 : public ImmutableSet<Change.Id> getChangeIds() {
151 53 : return changes.keySet();
152 : }
153 :
154 : public ImmutableSet<Change.Id> getChangeIds(BranchNameKey branch) {
155 53 : return byBranch.get(branch);
156 : }
157 :
158 : public CodeReviewCommit get(Change.Id changeId) {
159 53 : return commits.get(changeId);
160 : }
161 :
162 : public void put(CodeReviewCommit c) {
163 53 : commits.put(c.change().getId(), c);
164 53 : }
165 :
166 : public void problem(Change.Id id, String problem) {
167 16 : problems.put(id, problem);
168 16 : }
169 :
170 : public void logProblem(Change.Id id, Throwable t) {
171 0 : String msg = "Error reading change";
172 0 : logger.atSevere().withCause(t).log("%s %s", msg, id);
173 0 : problems.put(id, msg);
174 0 : }
175 :
176 : public void logProblem(Change.Id id, String msg) {
177 0 : logger.atSevere().log("%s %s", msg, id);
178 0 : problems.put(id, msg);
179 0 : }
180 :
181 : public boolean isOk() {
182 53 : return problems.isEmpty();
183 : }
184 :
185 : public List<SubmitRecord> getSubmitRecords(Change.Id id) {
186 : // Use the cached submit records from the original ChangeData in the input
187 : // ChangeSet, which were checked earlier in the integrate process. Even in
188 : // the case of a race where the submit records may have changed, it makes
189 : // more sense to store the original results of the submit rule evaluator
190 : // than to fail at this point.
191 : //
192 : // However, do NOT expose that ChangeData directly, as it is way out of
193 : // date by this point.
194 53 : ChangeData cd = requireNonNull(changes.get(id), () -> String.format("ChangeData for %s", id));
195 53 : return requireNonNull(
196 53 : cd.submitRecords(submitRuleOptions(allowClosed)),
197 : "getSubmitRecord only valid after submit rules are evalutated");
198 : }
199 :
200 : public void maybeFailVerbose() throws ResourceConflictException {
201 53 : if (isOk()) {
202 53 : return;
203 : }
204 16 : String msg =
205 : "Failed to submit "
206 16 : + changes.size()
207 : + " change"
208 16 : + (changes.size() > 1 ? "s" : "")
209 : + " due to the following problems:\n";
210 16 : List<String> ps = new ArrayList<>(problems.keySet().size());
211 16 : for (Change.Id id : problems.keySet()) {
212 16 : ps.add("Change " + id + ": " + Joiner.on("; ").join(problems.get(id)));
213 16 : }
214 16 : throw new ResourceConflictException(msg + Joiner.on('\n').join(ps));
215 : }
216 :
217 : public void maybeFail(String msgPrefix) throws ResourceConflictException {
218 53 : if (isOk()) {
219 53 : return;
220 : }
221 0 : StringBuilder msg = new StringBuilder(msgPrefix).append(" of change");
222 0 : Set<Change.Id> ids = problems.keySet();
223 0 : if (ids.size() == 1) {
224 0 : msg.append(" ").append(ids.iterator().next());
225 : } else {
226 0 : msg.append("s ").append(Joiner.on(", ").join(ids));
227 : }
228 0 : throw new ResourceConflictException(msg.toString());
229 : }
230 : }
231 :
232 : private final ChangeMessagesUtil cmUtil;
233 : private final BatchUpdate.Factory batchUpdateFactory;
234 : private final InternalUser.Factory internalUserFactory;
235 : private final MergeSuperSet mergeSuperSet;
236 : private final MergeValidators.Factory mergeValidatorsFactory;
237 : private final Provider<InternalChangeQuery> queryProvider;
238 : private final SubmitStrategyFactory submitStrategyFactory;
239 : private final SubscriptionGraph.Factory subscriptionGraphFactory;
240 : private final SubmoduleCommits.Factory submoduleCommitsFactory;
241 : private final ImmutableList<SubmissionListener> superprojectUpdateSubmissionListeners;
242 : private final Provider<MergeOpRepoManager> ormProvider;
243 : private final NotifyResolver notifyResolver;
244 : private final RetryHelper retryHelper;
245 : private final ChangeData.Factory changeDataFactory;
246 : private final StoreSubmitRequirementsOp.Factory storeSubmitRequirementsOpFactory;
247 :
248 : // Changes that were updated by this MergeOp.
249 : private final Map<Change.Id, Change> updatedChanges;
250 :
251 : private Instant ts;
252 : private SubmissionId submissionId;
253 : private IdentifiedUser caller;
254 :
255 : private MergeOpRepoManager orm;
256 : private CommitStatus commitStatus;
257 : private SubmitInput submitInput;
258 : private NotifyResolver.Result notify;
259 : private Set<Project.NameKey> projects;
260 : private boolean dryrun;
261 : private TopicMetrics topicMetrics;
262 :
263 : @Inject
264 : MergeOp(
265 : ChangeMessagesUtil cmUtil,
266 : BatchUpdate.Factory batchUpdateFactory,
267 : InternalUser.Factory internalUserFactory,
268 : MergeSuperSet mergeSuperSet,
269 : MergeValidators.Factory mergeValidatorsFactory,
270 : Provider<InternalChangeQuery> queryProvider,
271 : SubmitStrategyFactory submitStrategyFactory,
272 : SubmoduleCommits.Factory submoduleCommitsFactory,
273 : SubscriptionGraph.Factory subscriptionGraphFactory,
274 : @SuperprojectUpdateOnSubmission
275 : ImmutableList<SubmissionListener> superprojectUpdateSubmissionListeners,
276 : Provider<MergeOpRepoManager> ormProvider,
277 : NotifyResolver notifyResolver,
278 : TopicMetrics topicMetrics,
279 : RetryHelper retryHelper,
280 : ChangeData.Factory changeDataFactory,
281 53 : StoreSubmitRequirementsOp.Factory storeSubmitRequirementsOpFactory) {
282 53 : this.cmUtil = cmUtil;
283 53 : this.batchUpdateFactory = batchUpdateFactory;
284 53 : this.internalUserFactory = internalUserFactory;
285 53 : this.mergeSuperSet = mergeSuperSet;
286 53 : this.mergeValidatorsFactory = mergeValidatorsFactory;
287 53 : this.queryProvider = queryProvider;
288 53 : this.submitStrategyFactory = submitStrategyFactory;
289 53 : this.submoduleCommitsFactory = submoduleCommitsFactory;
290 53 : this.subscriptionGraphFactory = subscriptionGraphFactory;
291 53 : this.superprojectUpdateSubmissionListeners = superprojectUpdateSubmissionListeners;
292 53 : this.ormProvider = ormProvider;
293 53 : this.notifyResolver = notifyResolver;
294 53 : this.retryHelper = retryHelper;
295 53 : this.topicMetrics = topicMetrics;
296 53 : this.changeDataFactory = changeDataFactory;
297 53 : this.updatedChanges = new HashMap<>();
298 53 : this.storeSubmitRequirementsOpFactory = storeSubmitRequirementsOpFactory;
299 53 : }
300 :
301 : @Override
302 : public void close() {
303 53 : if (orm != null) {
304 53 : orm.close();
305 : }
306 53 : }
307 :
308 : public static void checkSubmitRequirements(ChangeData cd) throws ResourceConflictException {
309 69 : PatchSet patchSet = cd.currentPatchSet();
310 69 : if (patchSet == null) {
311 0 : throw new ResourceConflictException("missing current patch set for change " + cd.getId());
312 : }
313 69 : Map<SubmitRequirement, SubmitRequirementResult> srResults =
314 69 : cd.submitRequirementsIncludingLegacy();
315 69 : if (srResults.values().stream().allMatch(SubmitRequirementResult::fulfilled)) {
316 46 : return;
317 52 : } else if (srResults.isEmpty()) {
318 0 : throw new IllegalStateException(
319 0 : String.format(
320 : "Submit requirement results for change '%s' and patchset '%s' "
321 : + "are empty in project '%s'",
322 0 : cd.getId(), patchSet.id(), cd.change().getProject().get()));
323 : }
324 :
325 52 : for (SubmitRequirementResult srResult : srResults.values()) {
326 52 : switch (srResult.status()) {
327 : case SATISFIED:
328 : case NOT_APPLICABLE:
329 : case OVERRIDDEN:
330 : case FORCED:
331 5 : break;
332 :
333 : case ERROR:
334 1 : throw new ResourceConflictException(
335 1 : String.format(
336 : "submit requirement '%s' has an error: %s",
337 1 : srResult.submitRequirement().name(), srResult.errorMessage().orElse("")));
338 :
339 : case UNSATISFIED:
340 52 : throw new ResourceConflictException(
341 52 : String.format(
342 52 : "submit requirement '%s' is unsatisfied.", srResult.submitRequirement().name()));
343 :
344 : default:
345 0 : throw new IllegalStateException(
346 0 : String.format(
347 : "Unexpected submit requirement status %s for %s in %s",
348 0 : srResult.status().name(), patchSet.id().getId(), cd.change().getProject().get()));
349 : }
350 5 : }
351 0 : throw new IllegalStateException();
352 : }
353 :
354 : private static SubmitRuleOptions submitRuleOptions(boolean allowClosed) {
355 53 : return allowClosed ? SUBMIT_RULE_OPTIONS_ALLOW_CLOSED : SUBMIT_RULE_OPTIONS;
356 : }
357 :
358 : private static List<SubmitRecord> getSubmitRecords(ChangeData cd) {
359 8 : return cd.submitRecords(submitRuleOptions(/* allowClosed= */ false));
360 : }
361 :
362 : private void checkSubmitRulesAndState(ChangeSet cs, boolean allowMerged)
363 : throws ResourceConflictException {
364 46 : checkArgument(
365 46 : !cs.furtherHiddenChanges(), "checkSubmitRulesAndState called for topic with hidden change");
366 46 : for (ChangeData cd : cs.changes()) {
367 : try {
368 46 : if (!cd.change().isNew()) {
369 0 : if (!(cd.change().isMerged() && allowMerged)) {
370 0 : commitStatus.problem(
371 0 : cd.getId(), "Change " + cd.getId() + " is " + ChangeUtil.status(cd.change()));
372 : }
373 46 : } else if (cd.change().isWorkInProgress()) {
374 6 : commitStatus.problem(cd.getId(), "Change " + cd.getId() + " is work in progress");
375 : } else {
376 46 : checkSubmitRequirements(cd);
377 : }
378 5 : } catch (ResourceConflictException e) {
379 5 : commitStatus.problem(cd.getId(), e.getMessage());
380 0 : } catch (StorageException e) {
381 0 : String msg = "Error checking submit rules for change";
382 0 : logger.atWarning().withCause(e).log("%s %s", msg, cd.getId());
383 0 : commitStatus.problem(cd.getId(), msg);
384 46 : }
385 46 : }
386 46 : commitStatus.maybeFailVerbose();
387 46 : }
388 :
389 : private void bypassSubmitRulesAndRequirements(ChangeSet cs) {
390 8 : checkArgument(
391 8 : !cs.furtherHiddenChanges(), "cannot bypass submit rules for topic with hidden change");
392 8 : for (ChangeData cd : cs.changes()) {
393 8 : Change change = cd.change();
394 8 : if (change == null) {
395 0 : throw new StorageException("Change not found");
396 : }
397 8 : if (change.isClosed()) {
398 : // No need to check submit rules if the change is closed.
399 0 : continue;
400 : }
401 8 : List<SubmitRecord> records = new ArrayList<>(getSubmitRecords(cd));
402 8 : SubmitRecord forced = new SubmitRecord();
403 8 : forced.status = SubmitRecord.Status.FORCED;
404 8 : records.add(forced);
405 8 : cd.setSubmitRecords(submitRuleOptions(/* allowClosed= */ false), records);
406 :
407 : // Also bypass submit requirements. Mark them as forced.
408 8 : Map<SubmitRequirement, SubmitRequirementResult> forcedSRs =
409 8 : cd.submitRequirementsIncludingLegacy().entrySet().stream()
410 8 : .collect(
411 8 : Collectors.toMap(
412 : Map.Entry::getKey,
413 8 : entry -> entry.getValue().toBuilder().forced(Optional.of(true)).build()));
414 8 : cd.setSubmitRequirements(forcedSRs);
415 8 : }
416 8 : }
417 :
418 : /**
419 : * Merges the given change.
420 : *
421 : * <p>Depending on the server configuration, more changes may be affected, e.g. by submission of a
422 : * topic or via superproject subscriptions. All affected changes are integrated using the projects
423 : * integration strategy.
424 : *
425 : * @param change the change to be merged.
426 : * @param caller the identity of the caller
427 : * @param checkSubmitRules whether the prolog submit rules should be evaluated
428 : * @param submitInput parameters regarding the merge
429 : * @throws RestApiException if an error occurred.
430 : * @throws PermissionBackendException if permissions can't be checked
431 : * @throws IOException an error occurred reading from NoteDb.
432 : * @return the merged change
433 : */
434 : public Change merge(
435 : Change change,
436 : IdentifiedUser caller,
437 : boolean checkSubmitRules,
438 : SubmitInput submitInput,
439 : boolean dryrun)
440 : throws RestApiException, UpdateException, IOException, ConfigInvalidException,
441 : PermissionBackendException {
442 53 : this.submitInput = submitInput;
443 53 : this.notify =
444 53 : notifyResolver.resolve(
445 53 : firstNonNull(submitInput.notify, NotifyHandling.ALL), submitInput.notifyDetails);
446 53 : this.dryrun = dryrun;
447 53 : this.caller = caller;
448 53 : this.ts = TimeUtil.now();
449 53 : this.submissionId = new SubmissionId(change);
450 :
451 : try (TraceContext traceContext =
452 53 : TraceContext.open()
453 53 : .addTag(RequestId.Type.SUBMISSION_ID, new RequestId(submissionId.toString()))) {
454 53 : openRepoManager();
455 :
456 53 : logger.atFine().log("Beginning integration of %s", change);
457 : try {
458 53 : ChangeSet indexBackedChangeSet =
459 : mergeSuperSet
460 53 : .setMergeOpRepoManager(orm)
461 53 : .completeChangeSet(change, caller, /* includingTopicClosure= */ false);
462 53 : if (!indexBackedChangeSet.ids().contains(change.getId())) {
463 : // indexBackedChangeSet contains only open changes, if the change is missing in this set
464 : // it might be that the change was concurrently submitted in the meantime.
465 0 : change = changeDataFactory.create(change).reloadChange();
466 0 : if (!change.isNew()) {
467 0 : throw new ResourceConflictException("change is " + ChangeUtil.status(change));
468 : }
469 0 : throw new IllegalStateException(
470 0 : String.format("change %s missing from %s", change.getId(), indexBackedChangeSet));
471 : }
472 :
473 53 : if (indexBackedChangeSet.furtherHiddenChanges()) {
474 7 : throw new AuthException(
475 7 : "A change to be submitted with " + change.getId() + " is not visible");
476 : }
477 53 : logger.atFine().log("Calculated to merge %s", indexBackedChangeSet);
478 :
479 : // Reload ChangeSet so that we don't rely on (potentially) stale index data for merging
480 53 : ChangeSet noteDbChangeSet = reloadChanges(indexBackedChangeSet);
481 :
482 : // At this point, any change that isn't new can be filtered out since they were only here
483 : // in the first place due to stale index.
484 53 : List<ChangeData> filteredChanges = new ArrayList<>();
485 53 : for (ChangeData changeData : noteDbChangeSet.changes()) {
486 53 : if (!changeData.change().getStatus().equals(Status.NEW)) {
487 0 : logger.atFine().log(
488 : "Change %s has status %s due to stale index, so it is skipped during submit",
489 0 : changeData.getId(), changeData.change().getStatus().name());
490 0 : continue;
491 : }
492 53 : filteredChanges.add(changeData);
493 53 : }
494 :
495 : // There are no hidden changes (or else we would have thrown AuthException above).
496 53 : ChangeSet filteredNoteDbChangeSet =
497 53 : new ChangeSet(filteredChanges, /* hiddenChanges= */ ImmutableList.of());
498 :
499 : // Count cross-project submissions outside of the retry loop. The chance of a single project
500 : // failing increases with the number of projects, so the failure count would be inflated if
501 : // this metric were incremented inside of integrateIntoHistory.
502 53 : int projects = filteredNoteDbChangeSet.projects().size();
503 53 : if (projects > 1) {
504 11 : topicMetrics.topicSubmissions.increment();
505 : }
506 :
507 53 : SubmissionExecutor submissionExecutor =
508 : new SubmissionExecutor(dryrun, superprojectUpdateSubmissionListeners);
509 53 : RetryTracker retryTracker = new RetryTracker();
510 53 : retryHelper
511 53 : .changeUpdate(
512 : "integrateIntoHistory",
513 : updateFactory -> {
514 53 : long attempt = retryTracker.lastAttemptNumber + 1;
515 53 : boolean isRetry = attempt > 1;
516 53 : if (isRetry) {
517 9 : logger.atFine().log("Retrying, attempt #%d; skipping merged changes", attempt);
518 9 : this.ts = TimeUtil.now();
519 9 : openRepoManager();
520 : }
521 53 : this.commitStatus = new CommitStatus(filteredNoteDbChangeSet, isRetry);
522 53 : if (checkSubmitRules) {
523 46 : logger.atFine().log("Checking submit rules and state");
524 46 : checkSubmitRulesAndState(filteredNoteDbChangeSet, isRetry);
525 : } else {
526 8 : logger.atFine().log("Bypassing submit rules");
527 8 : bypassSubmitRulesAndRequirements(filteredNoteDbChangeSet);
528 : }
529 53 : integrateIntoHistory(
530 : filteredNoteDbChangeSet, submissionExecutor, checkSubmitRules);
531 53 : return null;
532 : })
533 53 : .listener(retryTracker)
534 : // Up to the entire submit operation is retried, including possibly many projects.
535 : // Multiply the timeout by the number of projects we're actually attempting to
536 : // submit. Times 2 to retry more persistently, to increase success rate.
537 53 : .defaultTimeoutMultiplier(filteredNoteDbChangeSet.projects().size() * 2)
538 : // By default, we only retry lock failures. Here it's better to also retry unexpected
539 : // runtime exceptions.
540 53 : .retryOn(t -> t instanceof RuntimeException)
541 53 : .call();
542 53 : submissionExecutor.afterExecutions(orm);
543 :
544 53 : if (projects > 1) {
545 11 : topicMetrics.topicSubmissionsCompleted.increment();
546 : }
547 :
548 : // It's expected that callers invoke this method only for open changes and that the provided
549 : // change either gets updated to merged or that this method fails with an exception. For
550 : // safety, fall-back to return the provided change if there was no update for this change
551 : // (e.g. caller provided a change that was already merged).
552 53 : return updatedChanges.containsKey(change.getId())
553 53 : ? updatedChanges.get(change.getId())
554 53 : : change;
555 0 : } catch (IOException e) {
556 : // Anything before the merge attempt is an error
557 0 : throw new StorageException(e);
558 : }
559 : }
560 : }
561 :
562 : private void openRepoManager() {
563 53 : if (orm != null) {
564 9 : orm.close();
565 : }
566 53 : orm = ormProvider.get();
567 53 : orm.setContext(ts, caller, notify);
568 53 : }
569 :
570 : private ChangeSet reloadChanges(ChangeSet changeSet) {
571 53 : List<ChangeData> visible = new ArrayList<>(changeSet.changes().size());
572 53 : List<ChangeData> nonVisible = new ArrayList<>(changeSet.nonVisibleChanges().size());
573 53 : changeSet.changes().forEach(c -> visible.add(changeDataFactory.create(c.project(), c.getId())));
574 53 : changeSet
575 53 : .nonVisibleChanges()
576 53 : .forEach(c -> nonVisible.add(changeDataFactory.create(c.project(), c.getId())));
577 53 : return new ChangeSet(visible, nonVisible);
578 : }
579 :
580 53 : private class RetryTracker implements RetryListener {
581 : long lastAttemptNumber;
582 :
583 : @Override
584 : public <V> void onRetry(Attempt<V> attempt) {
585 53 : lastAttemptNumber = attempt.getAttemptNumber();
586 53 : }
587 : }
588 :
589 : @Singleton
590 : private static class TopicMetrics {
591 : final Counter0 topicSubmissions;
592 : final Counter0 topicSubmissionsCompleted;
593 :
594 : @Inject
595 53 : TopicMetrics(MetricMaker metrics) {
596 53 : topicSubmissions =
597 53 : metrics.newCounter(
598 : "topic/cross_project_submit",
599 53 : new Description("Attempts at cross project topic submission").setRate());
600 53 : topicSubmissionsCompleted =
601 53 : metrics.newCounter(
602 : "topic/cross_project_submit_completed",
603 : new Description("Cross project topic submissions that concluded successfully")
604 53 : .setRate());
605 53 : }
606 : }
607 :
608 : private void integrateIntoHistory(
609 : ChangeSet cs, SubmissionExecutor submissionExecutor, boolean checkSubmitRules)
610 : throws RestApiException, UpdateException {
611 53 : checkArgument(!cs.furtherHiddenChanges(), "cannot integrate hidden changes into history");
612 53 : logger.atFine().log("Beginning merge attempt on %s", cs);
613 53 : Map<BranchNameKey, BranchBatch> toSubmit = new HashMap<>();
614 :
615 : ListMultimap<BranchNameKey, ChangeData> cbb;
616 : try {
617 53 : cbb = cs.changesByBranch();
618 0 : } catch (StorageException e) {
619 0 : throw new StorageException("Error reading changes to submit", e);
620 53 : }
621 53 : Set<BranchNameKey> branches = cbb.keySet();
622 :
623 53 : for (BranchNameKey branch : branches) {
624 53 : OpenRepo or = openRepo(branch.project());
625 53 : if (or != null) {
626 53 : toSubmit.put(branch, validateChangeList(or, cbb.get(branch)));
627 : }
628 53 : }
629 :
630 : // Done checks that don't involve running submit strategies.
631 53 : commitStatus.maybeFailVerbose();
632 :
633 : try {
634 53 : SubscriptionGraph subscriptionGraph = subscriptionGraphFactory.compute(branches, orm);
635 53 : SubmoduleCommits submoduleCommits = submoduleCommitsFactory.create(orm);
636 53 : UpdateOrderCalculator updateOrderCalculator = new UpdateOrderCalculator(subscriptionGraph);
637 53 : List<SubmitStrategy> strategies =
638 53 : getSubmitStrategies(
639 : toSubmit, updateOrderCalculator, submoduleCommits, subscriptionGraph, dryrun);
640 53 : this.projects = updateOrderCalculator.getProjectsInOrder();
641 53 : List<BatchUpdate> batchUpdates =
642 53 : orm.batchUpdates(
643 53 : projects, /* refLogMessage= */ checkSubmitRules ? "merged" : "forced-merge");
644 : // Group batch updates by project
645 53 : Map<Project.NameKey, BatchUpdate> batchUpdatesByProject =
646 53 : batchUpdates.stream().collect(Collectors.toMap(b -> b.getProject(), Function.identity()));
647 53 : for (Map.Entry<Change.Id, ChangeData> entry : cs.changesById().entrySet()) {
648 53 : Project.NameKey project = entry.getValue().project();
649 53 : Change.Id changeId = entry.getKey();
650 53 : ChangeData cd = entry.getValue();
651 53 : batchUpdatesByProject
652 53 : .get(project)
653 53 : .addOp(
654 : changeId,
655 53 : storeSubmitRequirementsOpFactory.create(
656 53 : cd.submitRequirementsIncludingLegacy().values(), cd));
657 53 : }
658 : try {
659 53 : submissionExecutor.setAdditionalBatchUpdateListeners(
660 53 : ImmutableList.of(new SubmitStrategyListener(submitInput, strategies, commitStatus)));
661 53 : submissionExecutor.execute(batchUpdates);
662 : } finally {
663 : // If the BatchUpdate fails it can be that merging some of the changes was actually
664 : // successful. This is why we must to collect the updated changes also when an
665 : // exception was thrown.
666 53 : strategies.forEach(s -> updatedChanges.putAll(s.getUpdatedChanges()));
667 :
668 : // Do not leave executed BatchUpdates in the OpenRepos
669 53 : if (!dryrun) {
670 53 : orm.resetUpdates(ImmutableSet.copyOf(this.projects));
671 : }
672 : }
673 0 : } catch (NoSuchProjectException e) {
674 0 : throw new ResourceNotFoundException(e.getMessage());
675 0 : } catch (IOException e) {
676 0 : throw new StorageException(e);
677 1 : } catch (SubmoduleConflictException e) {
678 1 : throw new IntegrationConflictException(e.getMessage(), e);
679 8 : } catch (UpdateException e) {
680 8 : if (e.getCause() instanceof LockFailureException) {
681 : // Lock failures are a special case: RetryHelper depends on this specific causal chain in
682 : // order to trigger a retry. The downside of throwing here is we will not get the nicer
683 : // error message constructed below, in the case where this is the final attempt and the
684 : // operation is not retried further. This is not a huge downside, and is hopefully so rare
685 : // as to be unnoticeable, assuming RetryHelper is retrying sufficiently.
686 8 : throw e;
687 : }
688 :
689 : // BatchUpdate may have inadvertently wrapped an IntegrationConflictException
690 : // thrown by some legacy SubmitStrategyOp code that intended the error
691 : // message to be user-visible. Copy the message from the wrapped
692 : // exception.
693 : //
694 : // If you happen across one of these, the correct fix is to convert the
695 : // inner IntegrationConflictException to a ResourceConflictException.
696 1 : if (e.getCause() instanceof IntegrationConflictException) {
697 0 : throw (IntegrationConflictException) e.getCause();
698 : }
699 1 : throw new MergeUpdateException(genericMergeError(cs), e);
700 53 : }
701 53 : }
702 :
703 : public Set<Project.NameKey> getAllProjects() {
704 0 : return projects;
705 : }
706 :
707 : public MergeOpRepoManager getMergeOpRepoManager() {
708 0 : return orm;
709 : }
710 :
711 : private List<SubmitStrategy> getSubmitStrategies(
712 : Map<BranchNameKey, BranchBatch> toSubmit,
713 : UpdateOrderCalculator updateOrderCalculator,
714 : SubmoduleCommits submoduleCommits,
715 : SubscriptionGraph subscriptionGraph,
716 : boolean dryrun)
717 : throws IntegrationConflictException, NoSuchProjectException, IOException {
718 53 : List<SubmitStrategy> strategies = new ArrayList<>();
719 53 : Set<BranchNameKey> allBranches = updateOrderCalculator.getBranchesInOrder();
720 53 : Set<CodeReviewCommit> allCommits =
721 53 : toSubmit.values().stream().map(BranchBatch::commits).flatMap(Set::stream).collect(toSet());
722 :
723 53 : for (BranchNameKey branch : allBranches) {
724 53 : OpenRepo or = orm.getRepo(branch.project());
725 53 : if (toSubmit.containsKey(branch)) {
726 53 : BranchBatch submitting = toSubmit.get(branch);
727 53 : logger.atFine().log("adding ops for branch %s, batch = %s", branch, submitting);
728 53 : OpenBranch ob = or.getBranch(branch);
729 53 : requireNonNull(
730 53 : submitting.submitType(),
731 53 : String.format("null submit type for %s; expected to previously fail fast", submitting));
732 53 : Set<CodeReviewCommit> commitsToSubmit = submitting.commits();
733 53 : ob.mergeTip = new MergeTip(ob.oldTip, commitsToSubmit);
734 53 : SubmitStrategy strategy =
735 53 : submitStrategyFactory.create(
736 53 : submitting.submitType(),
737 : or.rw,
738 : or.canMergeFlag,
739 53 : getAlreadyAccepted(or, ob.oldTip),
740 : allCommits,
741 : branch,
742 : caller,
743 : ob.mergeTip,
744 : commitStatus,
745 : submissionId,
746 : submitInput,
747 : submoduleCommits,
748 : subscriptionGraph,
749 : dryrun);
750 53 : strategies.add(strategy);
751 53 : strategy.addOps(or.getUpdate(), commitsToSubmit);
752 : }
753 53 : }
754 :
755 53 : return strategies;
756 : }
757 :
758 : private Set<RevCommit> getAlreadyAccepted(OpenRepo or, CodeReviewCommit branchTip) {
759 53 : Set<RevCommit> alreadyAccepted = new HashSet<>();
760 :
761 53 : if (branchTip != null) {
762 53 : alreadyAccepted.add(branchTip);
763 : }
764 :
765 : try {
766 53 : for (Ref r : or.repo.getRefDatabase().getRefsByPrefix(Constants.R_HEADS)) {
767 : try {
768 52 : CodeReviewCommit aac = or.rw.parseCommit(r.getObjectId());
769 52 : if (!commitStatus.commits.values().contains(aac)) {
770 52 : alreadyAccepted.add(aac);
771 : }
772 1 : } catch (IncorrectObjectTypeException iote) {
773 : // Not a commit? Skip over it.
774 52 : }
775 52 : }
776 0 : } catch (IOException e) {
777 0 : throw new StorageException("Failed to determine already accepted commits.", e);
778 53 : }
779 :
780 53 : logger.atFine().log("Found %d existing heads: %s", alreadyAccepted.size(), alreadyAccepted);
781 53 : return alreadyAccepted;
782 : }
783 :
784 : @AutoValue
785 53 : abstract static class BranchBatch {
786 : @Nullable
787 : abstract SubmitType submitType();
788 :
789 : abstract ImmutableSet<CodeReviewCommit> commits();
790 : }
791 :
792 : private BranchBatch validateChangeList(OpenRepo or, Collection<ChangeData> submitted) {
793 53 : logger.atFine().log("Validating %d changes", submitted.size());
794 53 : Set<CodeReviewCommit> toSubmit = new LinkedHashSet<>(submitted.size());
795 53 : SetMultimap<ObjectId, PatchSet.Id> revisions = getRevisions(or, submitted);
796 :
797 53 : SubmitType submitType = null;
798 53 : ChangeData choseSubmitTypeFrom = null;
799 53 : for (ChangeData cd : submitted) {
800 53 : Change.Id changeId = cd.getId();
801 : ChangeNotes notes;
802 : Change chg;
803 : SubmitType st;
804 : try {
805 53 : notes = cd.notes();
806 53 : chg = cd.change();
807 53 : st = getSubmitType(cd);
808 0 : } catch (StorageException e) {
809 0 : commitStatus.logProblem(changeId, e);
810 0 : continue;
811 53 : }
812 :
813 53 : if (st == null) {
814 0 : commitStatus.logProblem(changeId, "No submit type for change");
815 0 : continue;
816 : }
817 53 : if (submitType == null) {
818 53 : submitType = st;
819 53 : choseSubmitTypeFrom = cd;
820 17 : } else if (st != submitType) {
821 1 : commitStatus.problem(
822 : changeId,
823 1 : String.format(
824 : "Change has submit type %s, but previously chose submit type %s "
825 : + "from change %s in the same batch",
826 1 : st, submitType, choseSubmitTypeFrom.getId()));
827 1 : continue;
828 : }
829 53 : if (chg.currentPatchSetId() == null) {
830 0 : String msg = "Missing current patch set on change";
831 0 : logger.atSevere().log("%s %s", msg, changeId);
832 0 : commitStatus.problem(changeId, msg);
833 0 : continue;
834 : }
835 :
836 : PatchSet ps;
837 53 : BranchNameKey destBranch = chg.getDest();
838 : try {
839 53 : ps = cd.currentPatchSet();
840 0 : } catch (StorageException e) {
841 0 : commitStatus.logProblem(changeId, e);
842 0 : continue;
843 53 : }
844 53 : if (ps == null) {
845 0 : commitStatus.logProblem(changeId, "Missing patch set on change");
846 0 : continue;
847 : }
848 :
849 53 : ObjectId id = ps.commitId();
850 53 : if (!revisions.containsEntry(id, ps.id())) {
851 0 : if (revisions.containsValue(ps.id())) {
852 : // TODO This is actually an error, the patch set ref exists but points to a revision that
853 : // is different from the revision that we have stored for the patch set in the change
854 : // meta data.
855 0 : commitStatus.logProblem(
856 : changeId,
857 : "Revision "
858 0 : + id.name()
859 : + " of patch set "
860 0 : + ps.number()
861 : + " does not match the revision of the patch set ref "
862 0 : + ps.id().toRefName());
863 0 : continue;
864 : }
865 :
866 : // The patch set ref is not found but we want to merge the change. We can't safely do that
867 : // if the patch set ref is missing. In a cluster setups with multiple primary nodes this can
868 : // indicate a replication lag (e.g. the change meta data was already replicated, but the
869 : // replication of the patch set ref is still pending).
870 0 : commitStatus.logProblem(
871 : changeId,
872 : "Patch set ref "
873 0 : + ps.id().toRefName()
874 : + " not found. Expected patch set ref of "
875 0 : + ps.number()
876 : + " to point to revision "
877 0 : + id.name());
878 0 : continue;
879 : }
880 :
881 : CodeReviewCommit commit;
882 : try {
883 53 : commit = or.rw.parseCommit(id);
884 0 : } catch (IOException e) {
885 0 : commitStatus.logProblem(changeId, e);
886 0 : continue;
887 53 : }
888 :
889 53 : commit.setNotes(notes);
890 53 : commit.setPatchsetId(ps.id());
891 53 : commitStatus.put(commit);
892 :
893 53 : MergeValidators mergeValidators = mergeValidatorsFactory.create();
894 : try {
895 53 : mergeValidators.validatePreMerge(
896 53 : or.repo, or.rw, commit, or.project, destBranch, ps.id(), caller);
897 4 : } catch (MergeValidationException mve) {
898 4 : commitStatus.problem(changeId, mve.getMessage());
899 4 : continue;
900 53 : }
901 53 : commit.add(or.canMergeFlag);
902 53 : toSubmit.add(commit);
903 53 : }
904 53 : logger.atFine().log("Submitting on this run: %s", toSubmit);
905 53 : return new AutoValue_MergeOp_BranchBatch(submitType, ImmutableSet.copyOf(toSubmit));
906 : }
907 :
908 : private SetMultimap<ObjectId, PatchSet.Id> getRevisions(OpenRepo or, Collection<ChangeData> cds) {
909 : try {
910 53 : List<String> refNames = new ArrayList<>(cds.size());
911 53 : for (ChangeData cd : cds) {
912 53 : Change c = cd.change();
913 53 : if (c != null) {
914 53 : refNames.add(c.currentPatchSetId().toRefName());
915 : }
916 53 : }
917 53 : SetMultimap<ObjectId, PatchSet.Id> revisions =
918 53 : MultimapBuilder.hashKeys(cds.size()).hashSetValues(1).build();
919 : for (Map.Entry<String, Ref> e :
920 53 : or.repo
921 53 : .getRefDatabase()
922 53 : .exactRef(refNames.toArray(new String[refNames.size()]))
923 53 : .entrySet()) {
924 53 : revisions.put(e.getValue().getObjectId(), PatchSet.Id.fromRef(e.getKey()));
925 53 : }
926 53 : return revisions;
927 0 : } catch (IOException | StorageException e) {
928 0 : throw new StorageException("Failed to validate changes", e);
929 : }
930 : }
931 :
932 : @Nullable
933 : private SubmitType getSubmitType(ChangeData cd) {
934 53 : SubmitTypeRecord str = cd.submitTypeRecord();
935 53 : return str.isOk() ? str.type : null;
936 : }
937 :
938 : @Nullable
939 : private OpenRepo openRepo(Project.NameKey project) {
940 : try {
941 53 : return orm.getRepo(project);
942 0 : } catch (NoSuchProjectException e) {
943 0 : logger.atWarning().log("Project %s no longer exists, abandoning open changes.", project);
944 0 : abandonAllOpenChangeForDeletedProject(project);
945 0 : } catch (IOException e) {
946 0 : throw new StorageException("Error opening project " + project, e);
947 0 : }
948 0 : return null;
949 : }
950 :
951 : private void abandonAllOpenChangeForDeletedProject(Project.NameKey destProject) {
952 : try {
953 0 : for (ChangeData cd : queryProvider.get().byProjectOpen(destProject)) {
954 0 : try (BatchUpdate bu =
955 0 : batchUpdateFactory.create(destProject, internalUserFactory.create(), ts)) {
956 0 : bu.addOp(
957 0 : cd.getId(),
958 0 : new BatchUpdateOp() {
959 : @Override
960 : public boolean updateChange(ChangeContext ctx) {
961 0 : Change change = ctx.getChange();
962 0 : if (!change.isNew()) {
963 0 : return false;
964 : }
965 :
966 0 : change.setStatus(Change.Status.ABANDONED);
967 :
968 0 : cmUtil.setChangeMessage(
969 : ctx, "Project was deleted.", ChangeMessagesUtil.TAG_MERGED);
970 :
971 0 : return true;
972 : }
973 : });
974 : try {
975 0 : bu.execute();
976 0 : } catch (UpdateException | RestApiException e) {
977 0 : logger.atWarning().withCause(e).log(
978 : "Cannot abandon changes for deleted project %s", destProject);
979 0 : }
980 : }
981 0 : }
982 0 : } catch (StorageException e) {
983 0 : logger.atWarning().withCause(e).log(
984 : "Cannot abandon changes for deleted project %s", destProject);
985 0 : }
986 0 : }
987 :
988 : private String genericMergeError(ChangeSet cs) {
989 1 : int c = cs.size();
990 1 : if (c == 1) {
991 1 : return "Error submitting change";
992 : }
993 0 : int p = cs.projects().size();
994 0 : if (p == 1) {
995 : // Fused updates: it's correct to say that none of the n changes were submitted.
996 0 : return "Error submitting " + c + " changes";
997 : }
998 : // Multiple projects involved, but we don't know at this point what failed. At least give the
999 : // user a heads up that some changes may be unsubmitted, even if the change screen they land on
1000 : // after the error message says that this particular change was submitted.
1001 0 : return "Error submitting some of the "
1002 : + c
1003 : + " changes to one or more of the "
1004 : + p
1005 : + " projects involved; some projects may have submitted successfully, but others may have"
1006 : + " failed";
1007 : }
1008 : }
|
