Line data    Source code

       1             : // Copyright (C) 2017 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.index.project;
      16             : 
      17             : import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
      18             : import static com.google.gerrit.server.project.ProjectCache.illegalState;
      19             : 
      20             : import com.google.common.base.Stopwatch;
      21             : import com.google.common.flogger.FluentLogger;
      22             : import com.google.common.util.concurrent.Futures;
      23             : import com.google.common.util.concurrent.ListenableFuture;
      24             : import com.google.common.util.concurrent.ListeningExecutorService;
      25             : import com.google.gerrit.entities.Project;
      26             : import com.google.gerrit.index.SiteIndexer;
      27             : import com.google.gerrit.index.project.ProjectData;
      28             : import com.google.gerrit.index.project.ProjectIndex;
      29             : import com.google.gerrit.server.index.IndexExecutor;
      30             : import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
      31             : import com.google.gerrit.server.project.ProjectCache;
      32             : import com.google.inject.Inject;
      33             : import com.google.inject.Singleton;
      34             : import java.util.ArrayList;
      35             : import java.util.List;
      36             : import java.util.concurrent.ExecutionException;
      37             : import java.util.concurrent.atomic.AtomicBoolean;
      38             : import java.util.concurrent.atomic.AtomicInteger;
      39             : import org.eclipse.jgit.lib.ProgressMonitor;
      40             : import org.eclipse.jgit.lib.TextProgressMonitor;
      41             : 
      42             : /**
      43             :  * Implementation that can index all projects on a host. Used by Gerrit's initialization and upgrade
      44             :  * programs as well as by REST API endpoints that offer this functionality.
      45             :  */
      46             : @Singleton
      47             : public class AllProjectsIndexer extends SiteIndexer<Project.NameKey, ProjectData, ProjectIndex> {
      48         151 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      49             : 
      50             :   private final ListeningExecutorService executor;
      51             :   private final ProjectCache projectCache;
      52             :   private final IsFirstInsertForEntry isFirstInsertForEntry;
      53             : 
      54             :   @Inject
      55             :   AllProjectsIndexer(
      56             :       @IndexExecutor(BATCH) ListeningExecutorService executor,
      57             :       ProjectCache projectCache,
      58         151 :       IsFirstInsertForEntry isFirstInsertForEntry) {
      59         151 :     this.executor = executor;
      60         151 :     this.projectCache = projectCache;
      61         151 :     this.isFirstInsertForEntry = isFirstInsertForEntry;
      62         151 :   }
      63             : 
      64             :   @Override
      65             :   public SiteIndexer.Result indexAll(final ProjectIndex index) {
      66          15 :     ProgressMonitor progress = new TextProgressMonitor(newPrintWriter(progressOut));
      67          15 :     progress.start(2);
      68          15 :     List<Project.NameKey> names = collectProjects(progress);
      69          15 :     return reindexProjects(index, names, progress);
      70             :   }
      71             : 
      72             :   private SiteIndexer.Result reindexProjects(
      73             :       ProjectIndex index, List<Project.NameKey> names, ProgressMonitor progress) {
      74          15 :     progress.beginTask("Reindexing projects", names.size());
      75          15 :     List<ListenableFuture<?>> futures = new ArrayList<>(names.size());
      76          15 :     AtomicBoolean ok = new AtomicBoolean(true);
      77          15 :     AtomicInteger done = new AtomicInteger();
      78          15 :     AtomicInteger failed = new AtomicInteger();
      79          15 :     Stopwatch sw = Stopwatch.createStarted();
      80          15 :     for (Project.NameKey name : names) {
      81          15 :       String desc = "project " + name;
      82          15 :       ListenableFuture<?> future =
      83          15 :           executor.submit(
      84             :               () -> {
      85             :                 try {
      86          15 :                   projectCache.evict(name);
      87          15 :                   ProjectData projectData =
      88          15 :                       projectCache.get(name).orElseThrow(illegalState(name)).toProjectData();
      89          15 :                   if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
      90          15 :                     index.insert(projectData);
      91             :                   } else {
      92           0 :                     index.replace(projectData);
      93             :                   }
      94          15 :                   verboseWriter.println("Reindexed " + desc);
      95          15 :                   done.incrementAndGet();
      96           0 :                 } catch (Exception e) {
      97           0 :                   failed.incrementAndGet();
      98           0 :                   throw e;
      99          15 :                 }
     100          15 :                 return null;
     101             :               });
     102          15 :       addErrorListener(future, desc, progress, ok);
     103          15 :       futures.add(future);
     104          15 :     }
     105             : 
     106             :     try {
     107          15 :       Futures.successfulAsList(futures).get();
     108           0 :     } catch (ExecutionException | InterruptedException e) {
     109           0 :       logger.atSevere().withCause(e).log("Error waiting on project futures");
     110           0 :       return SiteIndexer.Result.create(sw, false, 0, 0);
     111          15 :     }
     112             : 
     113          15 :     progress.endTask();
     114          15 :     return SiteIndexer.Result.create(sw, ok.get(), done.get(), failed.get());
     115             :   }
     116             : 
     117             :   private List<Project.NameKey> collectProjects(ProgressMonitor progress) {
     118          15 :     progress.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
     119          15 :     List<Project.NameKey> names = new ArrayList<>();
     120          15 :     for (Project.NameKey nameKey : projectCache.all()) {
     121          15 :       names.add(nameKey);
     122          15 :     }
     123          15 :     progress.endTask();
     124          15 :     return names;
     125             :   }
     126             : }