Line data    Source code

       1             : // Copyright (C) 2018 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.project;
      16             : 
      17             : import static com.google.gerrit.index.query.Predicate.and;
      18             : import static com.google.gerrit.index.query.Predicate.or;
      19             : import static com.google.gerrit.server.query.change.ChangeStatusPredicate.open;
      20             : import static java.util.stream.Collectors.toSet;
      21             : 
      22             : import com.google.common.annotations.VisibleForTesting;
      23             : import com.google.common.base.Strings;
      24             : import com.google.common.base.Throwables;
      25             : import com.google.common.collect.ImmutableList;
      26             : import com.google.gerrit.entities.Change;
      27             : import com.google.gerrit.entities.PatchSet;
      28             : import com.google.gerrit.entities.Project;
      29             : import com.google.gerrit.entities.RefNames;
      30             : import com.google.gerrit.exceptions.StorageException;
      31             : import com.google.gerrit.extensions.api.changes.FixInput;
      32             : import com.google.gerrit.extensions.api.projects.CheckProjectInput;
      33             : import com.google.gerrit.extensions.api.projects.CheckProjectInput.AutoCloseableChangesCheckInput;
      34             : import com.google.gerrit.extensions.api.projects.CheckProjectResultInfo;
      35             : import com.google.gerrit.extensions.api.projects.CheckProjectResultInfo.AutoCloseableChangesCheckResult;
      36             : import com.google.gerrit.extensions.client.ListChangesOption;
      37             : import com.google.gerrit.extensions.common.ChangeInfo;
      38             : import com.google.gerrit.extensions.registration.DynamicItem;
      39             : import com.google.gerrit.extensions.restapi.BadRequestException;
      40             : import com.google.gerrit.extensions.restapi.ResourceConflictException;
      41             : import com.google.gerrit.extensions.restapi.RestApiException;
      42             : import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
      43             : import com.google.gerrit.index.IndexConfig;
      44             : import com.google.gerrit.index.query.Predicate;
      45             : import com.google.gerrit.server.ChangeUtil;
      46             : import com.google.gerrit.server.change.ChangeJson;
      47             : import com.google.gerrit.server.config.UrlFormatter;
      48             : import com.google.gerrit.server.git.GitRepositoryManager;
      49             : import com.google.gerrit.server.index.change.ChangeField;
      50             : import com.google.gerrit.server.query.change.ChangeData;
      51             : import com.google.gerrit.server.query.change.ChangePredicates;
      52             : import com.google.gerrit.server.update.RetryHelper;
      53             : import com.google.inject.Inject;
      54             : import com.google.inject.Singleton;
      55             : import java.io.IOException;
      56             : import java.util.ArrayList;
      57             : import java.util.HashMap;
      58             : import java.util.HashSet;
      59             : import java.util.List;
      60             : import java.util.Map;
      61             : import java.util.Set;
      62             : import org.eclipse.jgit.lib.ObjectId;
      63             : import org.eclipse.jgit.lib.Ref;
      64             : import org.eclipse.jgit.lib.Repository;
      65             : import org.eclipse.jgit.revwalk.RevCommit;
      66             : import org.eclipse.jgit.revwalk.RevSort;
      67             : import org.eclipse.jgit.revwalk.RevWalk;
      68             : 
      69             : @Singleton
      70             : public class ProjectsConsistencyChecker {
      71             :   @VisibleForTesting public static final int AUTO_CLOSE_MAX_COMMITS_LIMIT = 10000;
      72             : 
      73             :   private final GitRepositoryManager repoManager;
      74             :   private final RetryHelper retryHelper;
      75             :   private final ChangeJson.Factory changeJsonFactory;
      76             :   private final IndexConfig indexConfig;
      77             :   private final DynamicItem<UrlFormatter> urlFormatter;
      78             : 
      79             :   @Inject
      80             :   ProjectsConsistencyChecker(
      81             :       GitRepositoryManager repoManager,
      82             :       RetryHelper retryHelper,
      83             :       ChangeJson.Factory changeJsonFactory,
      84             :       IndexConfig indexConfig,
      85         146 :       DynamicItem<UrlFormatter> urlFormatter) {
      86         146 :     this.repoManager = repoManager;
      87         146 :     this.retryHelper = retryHelper;
      88         146 :     this.changeJsonFactory = changeJsonFactory;
      89         146 :     this.indexConfig = indexConfig;
      90         146 :     this.urlFormatter = urlFormatter;
      91         146 :   }
      92             : 
      93             :   public CheckProjectResultInfo check(Project.NameKey projectName, CheckProjectInput input)
      94             :       throws IOException, RestApiException {
      95           1 :     CheckProjectResultInfo r = new CheckProjectResultInfo();
      96           1 :     if (input.autoCloseableChangesCheck != null) {
      97           1 :       r.autoCloseableChangesCheckResult =
      98           1 :           checkForAutoCloseableChanges(projectName, input.autoCloseableChangesCheck);
      99             :     }
     100           1 :     return r;
     101             :   }
     102             : 
     103             :   private AutoCloseableChangesCheckResult checkForAutoCloseableChanges(
     104             :       Project.NameKey projectName, AutoCloseableChangesCheckInput input)
     105             :       throws IOException, RestApiException {
     106           1 :     AutoCloseableChangesCheckResult r = new AutoCloseableChangesCheckResult();
     107           1 :     if (Strings.isNullOrEmpty(input.branch)) {
     108           1 :       throw new BadRequestException("branch is required");
     109             :     }
     110             : 
     111           1 :     boolean fix = input.fix != null ? input.fix : false;
     112             : 
     113           1 :     if (input.maxCommits != null && input.maxCommits > AUTO_CLOSE_MAX_COMMITS_LIMIT) {
     114           1 :       throw new BadRequestException(
     115             :           "max commits can at most be set to " + AUTO_CLOSE_MAX_COMMITS_LIMIT);
     116             :     }
     117           1 :     int maxCommits = input.maxCommits != null ? input.maxCommits : AUTO_CLOSE_MAX_COMMITS_LIMIT;
     118             : 
     119             :     // Result that we want to return to the client.
     120           1 :     List<ChangeInfo> autoCloseableChanges = new ArrayList<>();
     121             : 
     122             :     // Remember the change IDs of all changes that we already included into the result, so that we
     123             :     // can avoid including the same change twice.
     124           1 :     Set<Change.Id> seenChanges = new HashSet<>();
     125             : 
     126           1 :     try (Repository repo = repoManager.openRepository(projectName);
     127           1 :         RevWalk rw = new RevWalk(repo)) {
     128           1 :       String branch = RefNames.fullName(input.branch);
     129           1 :       Ref ref = repo.exactRef(branch);
     130           1 :       if (ref == null) {
     131           1 :         throw new UnprocessableEntityException(
     132           1 :             String.format("branch '%s' not found", input.branch));
     133             :       }
     134             : 
     135           1 :       rw.reset();
     136           1 :       rw.markStart(rw.parseCommit(ref.getObjectId()));
     137           1 :       rw.sort(RevSort.TOPO);
     138           1 :       rw.sort(RevSort.REVERSE);
     139             : 
     140             :       // Cache the SHA1's of all merged commits. We need this for knowing which commit merged the
     141             :       // change when auto-closing changes by commit.
     142           1 :       List<ObjectId> mergedSha1s = new ArrayList<>();
     143             : 
     144             :       // Cache the Change-Id to commit SHA1 mapping for all Change-Id's that we find in merged
     145             :       // commits. We need this for knowing which commit merged the change when auto-closing
     146             :       // changes by Change-Id.
     147           1 :       Map<Change.Key, ObjectId> changeIdToMergedSha1 = new HashMap<>();
     148             : 
     149             :       // Base predicate which is fixed for every change query.
     150           1 :       Predicate<ChangeData> basePredicate =
     151           1 :           and(ChangePredicates.project(projectName), ChangePredicates.ref(branch), open());
     152             : 
     153           1 :       int maxLeafPredicates = indexConfig.maxTerms() - basePredicate.getLeafCount();
     154             : 
     155             :       // List of predicates by which we want to find open changes for the branch. These predicates
     156             :       // will be combined with the 'or' operator.
     157           1 :       List<Predicate<ChangeData>> predicates = new ArrayList<>(maxLeafPredicates);
     158             : 
     159             :       RevCommit commit;
     160           1 :       int skippedCommits = 0;
     161           1 :       int walkedCommits = 0;
     162           1 :       while ((commit = rw.next()) != null) {
     163           1 :         if (input.skipCommits != null && skippedCommits < input.skipCommits) {
     164           1 :           skippedCommits++;
     165           1 :           continue;
     166             :         }
     167             : 
     168           1 :         if (walkedCommits >= maxCommits) {
     169           1 :           break;
     170             :         }
     171           1 :         walkedCommits++;
     172             : 
     173           1 :         ObjectId commitId = commit.copy();
     174           1 :         mergedSha1s.add(commitId);
     175             : 
     176             :         // Consider all Change-Id lines since this is what ReceiveCommits#autoCloseChanges does.
     177           1 :         List<String> changeIds = ChangeUtil.getChangeIdsFromFooter(commit, urlFormatter.get());
     178             : 
     179             :         // Number of predicates that we need to add for this commit, 1 per Change-Id plus one for
     180             :         // the commit.
     181           1 :         int newPredicatesCount = changeIds.size() + 1;
     182             : 
     183             :         // We accumulated the max number of query terms that can be used in one query, execute
     184             :         // the query and start a new one.
     185           1 :         if (predicates.size() + newPredicatesCount > maxLeafPredicates) {
     186           0 :           autoCloseableChanges.addAll(
     187           0 :               executeQueryAndAutoCloseChanges(
     188             :                   basePredicate, seenChanges, predicates, fix, changeIdToMergedSha1, mergedSha1s));
     189           0 :           mergedSha1s.clear();
     190           0 :           changeIdToMergedSha1.clear();
     191           0 :           predicates.clear();
     192             : 
     193           0 :           if (newPredicatesCount > maxLeafPredicates) {
     194             :             // Whee, a single commit generates more than maxLeafPredicates predicates. Give up.
     195           0 :             throw new ResourceConflictException(
     196           0 :                 String.format(
     197           0 :                     "commit %s contains more Change-Ids than we can handle", commit.name()));
     198             :           }
     199             :         }
     200             : 
     201           1 :         changeIds.forEach(
     202             :             changeId -> {
     203             :               // It can happen that there are multiple merged commits with the same Change-Id
     204             :               // footer (e.g. if a change was cherry-picked to a stable branch stable branch which
     205             :               // then got merged back into master, or just by directly pushing several commits
     206             :               // with the same Change-Id). In this case it is hard to say which of the commits
     207             :               // should be used to auto-close an open change with the same Change-Id (and branch).
     208             :               // Possible approaches are:
     209             :               // 1. use the oldest commit with that Change-Id to auto-close the change
     210             :               // 2. use the newest commit with that Change-Id to auto-close the change
     211             :               // Possibility 1. has the disadvantage that the commit may have been merged before
     212             :               // the change was created in which case it is strange how it could auto-close the
     213             :               // change. Also this strategy would require to walk all commits since otherwise we
     214             :               // cannot be sure that we have seen the oldest commit with that Change-Id.
     215             :               // Possibility 2 has the disadvantage that it doesn't produce the same result as if
     216             :               // auto-closing on push would have worked, since on direct push the first commit with
     217             :               // a Change-Id of an open change would have closed that change. Also for this we
     218             :               // would need to consider all commits that are skipped.
     219             :               // Since both possibilities are not perfect and require extra effort we choose the
     220             :               // easiest approach, which is use the newest commit with that Change-Id that we have
     221             :               // seen (this means we ignore skipped commits). This should be okay since the
     222             :               // important thing for callers is that auto-closable changes are closed. Which of the
     223             :               // commits is used to auto-close a change if there are several candidates is of minor
     224             :               // importance and hence can be non-deterministic.
     225           1 :               Change.Key changeKey = Change.key(changeId);
     226           1 :               if (!changeIdToMergedSha1.containsKey(changeKey)) {
     227           1 :                 changeIdToMergedSha1.put(changeKey, commitId);
     228             :               }
     229             : 
     230             :               // Find changes that have a matching Change-Id.
     231           1 :               predicates.add(ChangePredicates.idPrefix(changeId));
     232           1 :             });
     233             : 
     234             :         // Find changes that have a matching commit.
     235           1 :         predicates.add(ChangePredicates.commitPrefix(commit.name()));
     236           1 :       }
     237             : 
     238           1 :       if (!predicates.isEmpty()) {
     239             :         // Execute the query with the remaining predicates that were collected.
     240           1 :         autoCloseableChanges.addAll(
     241           1 :             executeQueryAndAutoCloseChanges(
     242             :                 basePredicate, seenChanges, predicates, fix, changeIdToMergedSha1, mergedSha1s));
     243             :       }
     244             :     }
     245             : 
     246           1 :     r.autoCloseableChanges = autoCloseableChanges;
     247           1 :     return r;
     248             :   }
     249             : 
     250             :   private ImmutableList<ChangeInfo> executeQueryAndAutoCloseChanges(
     251             :       Predicate<ChangeData> basePredicate,
     252             :       Set<Change.Id> seenChanges,
     253             :       List<Predicate<ChangeData>> predicates,
     254             :       boolean fix,
     255             :       Map<Change.Key, ObjectId> changeIdToMergedSha1,
     256             :       List<ObjectId> mergedSha1s) {
     257           1 :     if (predicates.isEmpty()) {
     258           0 :       return ImmutableList.of();
     259             :     }
     260             : 
     261             :     try {
     262           1 :       List<ChangeData> queryResult =
     263             :           retryHelper
     264           1 :               .changeIndexQuery(
     265             :                   "projectsConsistencyCheckerQueryChanges",
     266             :                   q ->
     267           1 :                       q.setRequestedFields(ChangeField.CHANGE, ChangeField.PATCH_SET)
     268           1 :                           .query(and(basePredicate, or(predicates))))
     269           1 :               .call();
     270             : 
     271             :       // Result for this query that we want to return to the client.
     272           1 :       ImmutableList.Builder<ChangeInfo> autoCloseableChangesByBranch = ImmutableList.builder();
     273             : 
     274           1 :       for (ChangeData autoCloseableChange : queryResult) {
     275             :         // Skip changes that we have already processed, either by this query or by
     276             :         // earlier queries.
     277           1 :         if (seenChanges.add(autoCloseableChange.getId())) {
     278           1 :           retryHelper
     279           1 :               .changeUpdate(
     280             :                   "projectsConsistencyCheckerAutoCloseChanges",
     281             :                   () -> {
     282             :                     // Auto-close by change
     283           1 :                     if (changeIdToMergedSha1.containsKey(autoCloseableChange.change().getKey())) {
     284           1 :                       autoCloseableChangesByBranch.add(
     285           1 :                           changeJson(
     286           1 :                                   fix,
     287           1 :                                   changeIdToMergedSha1.get(autoCloseableChange.change().getKey()))
     288           1 :                               .format(autoCloseableChange));
     289           1 :                       return null;
     290             :                     }
     291             : 
     292             :                     // Auto-close by commit
     293             :                     for (ObjectId patchSetSha1 :
     294           0 :                         autoCloseableChange.patchSets().stream()
     295           0 :                             .map(PatchSet::commitId)
     296           0 :                             .collect(toSet())) {
     297           0 :                       if (mergedSha1s.contains(patchSetSha1)) {
     298           0 :                         autoCloseableChangesByBranch.add(
     299           0 :                             changeJson(fix, patchSetSha1).format(autoCloseableChange));
     300           0 :                         break;
     301             :                       }
     302           0 :                     }
     303           0 :                     return null;
     304             :                   })
     305           1 :               .call();
     306             :         }
     307           1 :       }
     308             : 
     309           1 :       return autoCloseableChangesByBranch.build();
     310           0 :     } catch (Exception e) {
     311           0 :       Throwables.throwIfUnchecked(e);
     312           0 :       throw new StorageException(e);
     313             :     }
     314             :   }
     315             : 
     316             :   private ChangeJson changeJson(Boolean fix, ObjectId mergedAs) {
     317           1 :     ChangeJson changeJson = changeJsonFactory.create(ListChangesOption.CHECK);
     318           1 :     if (fix != null && fix.booleanValue()) {
     319           1 :       FixInput fixInput = new FixInput();
     320           1 :       fixInput.expectMergedAs = mergedAs.name();
     321           1 :       changeJson.fix(fixInput);
     322             :     }
     323           1 :     return changeJson;
     324             :   }
     325             : }