Line data    Source code

       1             : // Copyright (C) 2017 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.index.group;
      16             : 
      17             : import static com.google.common.collect.ImmutableList.toImmutableList;
      18             : import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
      19             : 
      20             : import com.google.common.base.Stopwatch;
      21             : import com.google.common.flogger.FluentLogger;
      22             : import com.google.common.util.concurrent.Futures;
      23             : import com.google.common.util.concurrent.ListenableFuture;
      24             : import com.google.common.util.concurrent.ListeningExecutorService;
      25             : import com.google.gerrit.entities.AccountGroup;
      26             : import com.google.gerrit.entities.GroupReference;
      27             : import com.google.gerrit.entities.InternalGroup;
      28             : import com.google.gerrit.index.SiteIndexer;
      29             : import com.google.gerrit.server.account.GroupCache;
      30             : import com.google.gerrit.server.group.db.Groups;
      31             : import com.google.gerrit.server.group.db.GroupsNoteDbConsistencyChecker;
      32             : import com.google.gerrit.server.index.IndexExecutor;
      33             : import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
      34             : import com.google.inject.Inject;
      35             : import com.google.inject.Singleton;
      36             : import java.io.IOException;
      37             : import java.util.ArrayList;
      38             : import java.util.List;
      39             : import java.util.Map;
      40             : import java.util.concurrent.ExecutionException;
      41             : import java.util.concurrent.atomic.AtomicBoolean;
      42             : import java.util.concurrent.atomic.AtomicInteger;
      43             : import org.eclipse.jgit.errors.ConfigInvalidException;
      44             : import org.eclipse.jgit.lib.ProgressMonitor;
      45             : import org.eclipse.jgit.lib.TextProgressMonitor;
      46             : 
      47             : /**
      48             :  * Implementation that can index all groups on a host. Used by Gerrit's initialization and upgrade
      49             :  * programs as well as by REST API endpoints that offer this functionality.
      50             :  */
      51             : @Singleton
      52             : public class AllGroupsIndexer extends SiteIndexer<AccountGroup.UUID, InternalGroup, GroupIndex> {
      53         138 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      54             : 
      55             :   private final ListeningExecutorService executor;
      56             :   private final GroupCache groupCache;
      57             :   private final Groups groups;
      58             :   private final IsFirstInsertForEntry isFirstInsertForEntry;
      59             : 
      60             :   @Inject
      61             :   AllGroupsIndexer(
      62             :       @IndexExecutor(BATCH) ListeningExecutorService executor,
      63             :       GroupCache groupCache,
      64             :       Groups groups,
      65         138 :       IsFirstInsertForEntry isFirstInsertForEntry) {
      66         138 :     this.executor = executor;
      67         138 :     this.groupCache = groupCache;
      68         138 :     this.groups = groups;
      69         138 :     this.isFirstInsertForEntry = isFirstInsertForEntry;
      70         138 :   }
      71             : 
      72             :   @Override
      73             :   public SiteIndexer.Result indexAll(GroupIndex index) {
      74          15 :     ProgressMonitor progress = new TextProgressMonitor(newPrintWriter(progressOut));
      75          15 :     progress.start(2);
      76          15 :     Stopwatch sw = Stopwatch.createStarted();
      77             :     List<AccountGroup.UUID> uuids;
      78             :     try {
      79          15 :       uuids = collectGroups(progress);
      80           0 :     } catch (IOException | ConfigInvalidException e) {
      81           0 :       logger.atSevere().withCause(e).log("Error collecting groups");
      82           0 :       return SiteIndexer.Result.create(sw, false, 0, 0);
      83          15 :     }
      84          15 :     return reindexGroups(index, uuids, progress);
      85             :   }
      86             : 
      87             :   private SiteIndexer.Result reindexGroups(
      88             :       GroupIndex index, List<AccountGroup.UUID> uuids, ProgressMonitor progress) {
      89          15 :     progress.beginTask("Reindexing groups", uuids.size());
      90          15 :     List<ListenableFuture<?>> futures = new ArrayList<>(uuids.size());
      91          15 :     AtomicBoolean ok = new AtomicBoolean(true);
      92          15 :     AtomicInteger done = new AtomicInteger();
      93          15 :     AtomicInteger failed = new AtomicInteger();
      94          15 :     Stopwatch sw = Stopwatch.createStarted();
      95          15 :     groupCache.evict(uuids);
      96          15 :     Map<AccountGroup.UUID, InternalGroup> reindexedGroups = groupCache.get(uuids);
      97          15 :     for (AccountGroup.UUID uuid : uuids) {
      98          15 :       String desc = "group " + uuid;
      99          15 :       ListenableFuture<?> future =
     100          15 :           executor.submit(
     101             :               () -> {
     102             :                 try {
     103          15 :                   groupCache.evict(uuid);
     104          15 :                   InternalGroup internalGroup = reindexedGroups.get(uuid);
     105          15 :                   if (internalGroup != null) {
     106          15 :                     if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
     107          15 :                       index.insert(internalGroup);
     108             :                     } else {
     109           0 :                       index.replace(internalGroup);
     110             :                     }
     111             :                   } else {
     112           0 :                     index.delete(uuid);
     113             : 
     114             :                     // The UUID here is read from group name notes. If it fails to load from group
     115             :                     // cache, there exists an inconsistency.
     116           0 :                     GroupsNoteDbConsistencyChecker.logFailToLoadFromGroupRefAsWarning(uuid);
     117             :                   }
     118          15 :                   verboseWriter.println("Reindexed " + desc);
     119          15 :                   done.incrementAndGet();
     120           0 :                 } catch (Exception e) {
     121           0 :                   failed.incrementAndGet();
     122           0 :                   throw e;
     123          15 :                 }
     124          15 :                 return null;
     125             :               });
     126          15 :       addErrorListener(future, desc, progress, ok);
     127          15 :       futures.add(future);
     128          15 :     }
     129             : 
     130             :     try {
     131          15 :       Futures.successfulAsList(futures).get();
     132           0 :     } catch (ExecutionException | InterruptedException e) {
     133           0 :       logger.atSevere().withCause(e).log("Error waiting on group futures");
     134           0 :       return SiteIndexer.Result.create(sw, false, 0, 0);
     135          15 :     }
     136             : 
     137          15 :     progress.endTask();
     138          15 :     return SiteIndexer.Result.create(sw, ok.get(), done.get(), failed.get());
     139             :   }
     140             : 
     141             :   private List<AccountGroup.UUID> collectGroups(ProgressMonitor progress)
     142             :       throws IOException, ConfigInvalidException {
     143          15 :     progress.beginTask("Collecting groups", ProgressMonitor.UNKNOWN);
     144             :     try {
     145          15 :       return groups.getAllGroupReferences().map(GroupReference::getUUID).collect(toImmutableList());
     146             :     } finally {
     147          15 :       progress.endTask();
     148             :     }
     149             :   }
     150             : }