Line data    Source code

       1             : // Copyright (C) 2019 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.account.externalids;
      16             : 
      17             : import com.google.common.base.CharMatcher;
      18             : import com.google.common.cache.Cache;
      19             : import com.google.common.collect.ImmutableMap;
      20             : import com.google.common.collect.ImmutableSet;
      21             : import com.google.common.collect.ImmutableSetMultimap;
      22             : import com.google.common.flogger.FluentLogger;
      23             : import com.google.gerrit.entities.Account;
      24             : import com.google.gerrit.entities.RefNames;
      25             : import com.google.gerrit.metrics.Counter1;
      26             : import com.google.gerrit.metrics.Description;
      27             : import com.google.gerrit.metrics.Description.Units;
      28             : import com.google.gerrit.metrics.Field;
      29             : import com.google.gerrit.metrics.MetricMaker;
      30             : import com.google.gerrit.metrics.Timer0;
      31             : import com.google.gerrit.server.config.AllUsersName;
      32             : import com.google.gerrit.server.config.GerritServerConfig;
      33             : import com.google.gerrit.server.git.GitRepositoryManager;
      34             : import com.google.gerrit.server.logging.Metadata;
      35             : import com.google.gerrit.server.logging.TraceContext;
      36             : import com.google.gerrit.server.logging.TraceContext.TraceTimer;
      37             : import com.google.inject.Inject;
      38             : import com.google.inject.Singleton;
      39             : import com.google.inject.name.Named;
      40             : import java.io.IOException;
      41             : import java.util.HashMap;
      42             : import java.util.HashSet;
      43             : import java.util.Map;
      44             : import java.util.Set;
      45             : import java.util.concurrent.TimeUnit;
      46             : import org.eclipse.jgit.errors.ConfigInvalidException;
      47             : import org.eclipse.jgit.lib.Config;
      48             : import org.eclipse.jgit.lib.ObjectId;
      49             : import org.eclipse.jgit.lib.ObjectReader;
      50             : import org.eclipse.jgit.lib.Ref;
      51             : import org.eclipse.jgit.lib.Repository;
      52             : import org.eclipse.jgit.revwalk.RevCommit;
      53             : import org.eclipse.jgit.revwalk.RevWalk;
      54             : import org.eclipse.jgit.treewalk.TreeWalk;
      55             : import org.eclipse.jgit.treewalk.filter.TreeFilter;
      56             : 
      57             : /** Loads cache values for the external ID cache using either a full or a partial reload. */
      58             : @Singleton
      59             : public class ExternalIdCacheLoader {
      60         152 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      61             : 
      62             :   // Maximum number of prior states we inspect to find a base for differential. If no cached state
      63             :   // is found within this number of parents, we fall back to reading everything from scratch.
      64             :   private static final int MAX_HISTORY_LOOKBACK = 10;
      65             : 
      66             :   private final ExternalIdReader externalIdReader;
      67             :   private final Cache<ObjectId, AllExternalIds> externalIdCache;
      68             :   private final GitRepositoryManager gitRepositoryManager;
      69             :   private final AllUsersName allUsersName;
      70             :   private final Counter1<Boolean> reloadCounter;
      71             :   private final Timer0 reloadDifferential;
      72             :   private final boolean isPersistentCache;
      73             :   private final ExternalIdFactory externalIdFactory;
      74             : 
      75             :   @Inject
      76             :   ExternalIdCacheLoader(
      77             :       GitRepositoryManager gitRepositoryManager,
      78             :       AllUsersName allUsersName,
      79             :       ExternalIdReader externalIdReader,
      80             :       @Named(ExternalIdCacheImpl.CACHE_NAME) Cache<ObjectId, AllExternalIds> externalIdCache,
      81             :       MetricMaker metricMaker,
      82             :       @GerritServerConfig Config config,
      83         152 :       ExternalIdFactory externalIdFactory) {
      84         152 :     this.externalIdReader = externalIdReader;
      85         152 :     this.externalIdCache = externalIdCache;
      86         152 :     this.gitRepositoryManager = gitRepositoryManager;
      87         152 :     this.allUsersName = allUsersName;
      88         152 :     this.reloadCounter =
      89         152 :         metricMaker.newCounter(
      90             :             "notedb/external_id_cache_load_count",
      91             :             new Description("Total number of external ID cache reloads from Git.")
      92         152 :                 .setRate()
      93         152 :                 .setUnit("updates"),
      94         152 :             Field.ofBoolean("partial", Metadata.Builder::partial)
      95         152 :                 .description("Whether the reload was partial.")
      96         152 :                 .build());
      97         152 :     this.reloadDifferential =
      98         152 :         metricMaker.newTimer(
      99             :             "notedb/external_id_partial_read_latency",
     100             :             new Description(
     101             :                     "Latency for generating a new external ID cache state from a prior state.")
     102         152 :                 .setCumulative()
     103         152 :                 .setUnit(Units.MILLISECONDS));
     104         152 :     this.isPersistentCache =
     105         152 :         config.getInt("cache", ExternalIdCacheImpl.CACHE_NAME, "diskLimit", 0) > 0;
     106         152 :     this.externalIdFactory = externalIdFactory;
     107         152 :   }
     108             : 
     109             :   public AllExternalIds load(ObjectId notesRev) throws IOException, ConfigInvalidException {
     110         151 :     externalIdReader.checkReadEnabled();
     111             :     // The requested value was not in the cache (hence, this loader was invoked). Therefore, try to
     112             :     // create this entry from a past value using the minimal amount of Git operations possible to
     113             :     // reduce latency.
     114             :     //
     115             :     // First, try to find the most recent state we have in the cache. Most of the time, this will be
     116             :     // the state before the last update happened, but it can also date further back. We try a best
     117             :     // effort approach and check the last 10 states. If nothing is found, we default to loading the
     118             :     // value from scratch.
     119             :     //
     120             :     // If a prior state was found, we use Git to diff the trees and find modifications. This is
     121             :     // faster than just loading the complete current tree and working off of that because of how the
     122             :     // data is structured: NotesMaps use nested trees, so, for example, a NotesMap with 200k entries
     123             :     // has two layers of nesting: 12/34/1234..99. TreeWalk is smart in skipping the traversal of
     124             :     // identical subtrees.
     125             :     //
     126             :     // Once we know what files changed, we apply additions and removals to the previously cached
     127             :     // state.
     128             : 
     129         151 :     try (Repository repo = gitRepositoryManager.openRepository(allUsersName);
     130         151 :         RevWalk rw = new RevWalk(repo)) {
     131         151 :       long start = System.nanoTime();
     132         151 :       Ref extIdRef = repo.exactRef(RefNames.REFS_EXTERNAL_IDS);
     133         151 :       if (extIdRef == null) {
     134          15 :         logger.atInfo().log(
     135             :             RefNames.REFS_EXTERNAL_IDS + " not initialized, falling back to full reload.");
     136          15 :         return reloadAllExternalIds(notesRev);
     137             :       }
     138             : 
     139         151 :       RevCommit currentCommit = rw.parseCommit(extIdRef.getObjectId());
     140         151 :       rw.markStart(currentCommit);
     141             :       RevCommit parentWithCacheValue;
     142         151 :       AllExternalIds oldExternalIds = null;
     143         151 :       int i = 0;
     144         151 :       while ((parentWithCacheValue = rw.next()) != null
     145             :           && i++ < MAX_HISTORY_LOOKBACK
     146         151 :           && parentWithCacheValue.getParentCount() < 2) {
     147         151 :         oldExternalIds = externalIdCache.getIfPresent(parentWithCacheValue.getId());
     148         151 :         if (oldExternalIds != null) {
     149             :           // We found a previously cached state.
     150         147 :           break;
     151             :         }
     152             :       }
     153         151 :       if (oldExternalIds == null) {
     154         151 :         if (isPersistentCache) {
     155             :           // If there is no persistence, this is normal. Don't upset admins reading the logs.
     156           0 :           logger.atWarning().log(
     157             :               "Unable to find an old ExternalId cache state, falling back to full reload");
     158             :         }
     159         151 :         return reloadAllExternalIds(notesRev);
     160             :       }
     161             : 
     162             :       // Diff trees to recognize modifications
     163         147 :       Set<ObjectId> removals = new HashSet<>(); // Set<Blob-Object-Id>
     164         147 :       Map<ObjectId, ObjectId> additions = new HashMap<>(); // Map<Name-ObjectId, Blob-Object-Id>
     165         147 :       try (TreeWalk treeWalk = new TreeWalk(repo)) {
     166         147 :         treeWalk.setFilter(TreeFilter.ANY_DIFF);
     167         147 :         treeWalk.setRecursive(true);
     168         147 :         treeWalk.reset(parentWithCacheValue.getTree(), currentCommit.getTree());
     169         147 :         while (treeWalk.next()) {
     170         147 :           String path = treeWalk.getPathString();
     171         147 :           ObjectId oldBlob = treeWalk.getObjectId(0);
     172         147 :           ObjectId newBlob = treeWalk.getObjectId(1);
     173         147 :           if (ObjectId.zeroId().equals(newBlob)) {
     174             :             // Deletion
     175           6 :             removals.add(oldBlob);
     176         147 :           } else if (ObjectId.zeroId().equals(oldBlob)) {
     177             :             // Addition
     178         147 :             additions.put(fileNameToObjectId(path), newBlob);
     179             :           } else {
     180             :             // Modification
     181          12 :             removals.add(oldBlob);
     182          12 :             additions.put(fileNameToObjectId(path), newBlob);
     183             :           }
     184         147 :         }
     185             :       }
     186             : 
     187         147 :       AllExternalIds allExternalIds =
     188         147 :           buildAllExternalIds(repo, oldExternalIds, additions, removals);
     189         147 :       reloadCounter.increment(true);
     190         147 :       reloadDifferential.record(System.nanoTime() - start, TimeUnit.NANOSECONDS);
     191         147 :       return allExternalIds;
     192         151 :     }
     193             :   }
     194             : 
     195             :   private static ObjectId fileNameToObjectId(String path) {
     196         147 :     return ObjectId.fromString(CharMatcher.is('/').removeFrom(path));
     197             :   }
     198             : 
     199             :   /**
     200             :    * Build a new {@link AllExternalIds} from an old state by applying additions and removals that
     201             :    * were performed since then.
     202             :    *
     203             :    * <p>Removals are applied before additions.
     204             :    *
     205             :    * @param repo open repository
     206             :    * @param oldExternalIds prior state that is used as base
     207             :    * @param additions map of name to blob ID for each external ID that should be added
     208             :    * @param removals set of name {@link ObjectId}s that should be removed
     209             :    */
     210             :   private AllExternalIds buildAllExternalIds(
     211             :       Repository repo,
     212             :       AllExternalIds oldExternalIds,
     213             :       Map<ObjectId, ObjectId> additions,
     214             :       Set<ObjectId> removals)
     215             :       throws IOException {
     216         147 :     ImmutableMap.Builder<ExternalId.Key, ExternalId> byKey = ImmutableMap.builder();
     217         147 :     ImmutableSetMultimap.Builder<Account.Id, ExternalId> byAccount = ImmutableSetMultimap.builder();
     218         147 :     ImmutableSetMultimap.Builder<String, ExternalId> byEmail = ImmutableSetMultimap.builder();
     219             : 
     220             :     // Copy over old ExternalIds but exclude deleted ones
     221         147 :     for (ExternalId externalId : oldExternalIds.byAccount().values()) {
     222         147 :       if (removals.contains(externalId.blobId())) {
     223          15 :         continue;
     224             :       }
     225             : 
     226         147 :       byKey.put(externalId.key(), externalId);
     227         147 :       byAccount.put(externalId.accountId(), externalId);
     228         147 :       if (externalId.email() != null) {
     229         143 :         byEmail.put(externalId.email(), externalId);
     230             :       }
     231         147 :     }
     232             : 
     233             :     // Add newly discovered ExternalIds
     234         147 :     try (ObjectReader reader = repo.newObjectReader()) {
     235         147 :       for (Map.Entry<ObjectId, ObjectId> nameToBlob : additions.entrySet()) {
     236             :         ExternalId parsedExternalId;
     237             :         try {
     238         147 :           parsedExternalId =
     239         147 :               externalIdFactory.parse(
     240         147 :                   nameToBlob.getKey().name(),
     241         147 :                   reader.open(nameToBlob.getValue()).getCachedBytes(),
     242         147 :                   nameToBlob.getValue());
     243           3 :         } catch (ConfigInvalidException | RuntimeException e) {
     244           3 :           logger.atSevere().withCause(e).log(
     245           3 :               "Ignoring invalid external ID note %s", nameToBlob.getKey().name());
     246           3 :           continue;
     247         147 :         }
     248             : 
     249         147 :         byKey.put(parsedExternalId.key(), parsedExternalId);
     250         147 :         byAccount.put(parsedExternalId.accountId(), parsedExternalId);
     251         147 :         if (parsedExternalId.email() != null) {
     252         146 :           byEmail.put(parsedExternalId.email(), parsedExternalId);
     253             :         }
     254         147 :       }
     255             :     }
     256         147 :     return AllExternalIds.create(byKey.build(), byAccount.build(), byEmail.build());
     257             :   }
     258             : 
     259             :   private AllExternalIds reloadAllExternalIds(ObjectId notesRev)
     260             :       throws IOException, ConfigInvalidException {
     261         151 :     try (TraceTimer ignored =
     262         151 :         TraceContext.newTimer(
     263             :             "Loading external IDs from scratch",
     264         151 :             Metadata.builder().revision(notesRev.name()).build())) {
     265         151 :       ImmutableSet<ExternalId> externalIds = externalIdReader.all(notesRev);
     266         151 :       externalIds.forEach(ExternalId::checkThatBlobIdIsSet);
     267         151 :       AllExternalIds allExternalIds = AllExternalIds.create(externalIds.stream());
     268         151 :       reloadCounter.increment(false);
     269         151 :       return allExternalIds;
     270             :     }
     271             :   }
     272             : }