Line data    Source code

       1             : // Copyright (C) 2009 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.account;
      16             : 
      17             : import static com.google.common.collect.ImmutableMap.toImmutableMap;
      18             : import static com.google.common.collect.ImmutableSet.toImmutableSet;
      19             : 
      20             : import com.google.common.cache.CacheLoader;
      21             : import com.google.common.cache.LoadingCache;
      22             : import com.google.common.collect.ImmutableMap;
      23             : import com.google.common.collect.ImmutableSet;
      24             : import com.google.common.collect.Iterables;
      25             : import com.google.common.flogger.FluentLogger;
      26             : import com.google.gerrit.common.Nullable;
      27             : import com.google.gerrit.entities.AccountGroup;
      28             : import com.google.gerrit.entities.InternalGroup;
      29             : import com.google.gerrit.entities.RefNames;
      30             : import com.google.gerrit.proto.Protos;
      31             : import com.google.gerrit.server.cache.CacheModule;
      32             : import com.google.gerrit.server.cache.proto.Cache;
      33             : import com.google.gerrit.server.cache.serialize.CacheSerializer;
      34             : import com.google.gerrit.server.cache.serialize.ObjectIdConverter;
      35             : import com.google.gerrit.server.cache.serialize.ProtobufSerializer;
      36             : import com.google.gerrit.server.cache.serialize.entities.InternalGroupSerializer;
      37             : import com.google.gerrit.server.config.AllUsersName;
      38             : import com.google.gerrit.server.git.GitRepositoryManager;
      39             : import com.google.gerrit.server.group.db.Groups;
      40             : import com.google.gerrit.server.logging.Metadata;
      41             : import com.google.gerrit.server.logging.TraceContext;
      42             : import com.google.gerrit.server.logging.TraceContext.TraceTimer;
      43             : import com.google.gerrit.server.query.group.InternalGroupQuery;
      44             : import com.google.inject.Inject;
      45             : import com.google.inject.Module;
      46             : import com.google.inject.Provider;
      47             : import com.google.inject.Singleton;
      48             : import com.google.inject.TypeLiteral;
      49             : import com.google.inject.name.Named;
      50             : import java.util.ArrayList;
      51             : import java.util.Collection;
      52             : import java.util.HashMap;
      53             : import java.util.Iterator;
      54             : import java.util.List;
      55             : import java.util.Map;
      56             : import java.util.Optional;
      57             : import java.util.Set;
      58             : import java.util.concurrent.ExecutionException;
      59             : import org.bouncycastle.util.Strings;
      60             : import org.eclipse.jgit.lib.ObjectId;
      61             : import org.eclipse.jgit.lib.Ref;
      62             : import org.eclipse.jgit.lib.Repository;
      63             : 
      64             : /** Tracks group objects in memory for efficient access. */
      65             : @Singleton
      66             : public class GroupCacheImpl implements GroupCache {
      67         152 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      68             : 
      69             :   private static final String BYID_NAME = "groups";
      70             :   private static final String BYNAME_NAME = "groups_byname";
      71             :   private static final String BYUUID_NAME = "groups_byuuid";
      72             :   private static final String BYUUID_NAME_PERSISTED = "groups_byuuid_persisted";
      73             : 
      74             :   public static Module module() {
      75         152 :     return new CacheModule() {
      76             :       @Override
      77             :       protected void configure() {
      78         152 :         cache(BYID_NAME, AccountGroup.Id.class, new TypeLiteral<Optional<InternalGroup>>() {})
      79         152 :             .maximumWeight(Long.MAX_VALUE)
      80         152 :             .loader(ByIdLoader.class);
      81             : 
      82         152 :         cache(BYNAME_NAME, String.class, new TypeLiteral<Optional<InternalGroup>>() {})
      83         152 :             .maximumWeight(Long.MAX_VALUE)
      84         152 :             .loader(ByNameLoader.class);
      85             : 
      86             :         // We split the group cache into two parts for performance reasons:
      87             :         // 1) An in-memory part that has only the group ref uuid as key.
      88             :         // 2) A persisted part that has the group ref uuid and sha1 of the ref as key.
      89             :         //
      90             :         // When loading dashboards or returning change query results we potentially
      91             :         // need to access many groups.
      92             :         // We want the persisted cache to be immutable and we want it to be impossible that a
      93             :         // value for a given key is out of date. We therefore require the sha-1 in the key. That
      94             :         // is in line with the rest of the caches in Gerrit.
      95             :         //
      96             :         // Splitting the cache into two chunks internally in this class allows us to retain
      97             :         // the existing performance guarantees of not requiring reads for the repo for values
      98             :         // cached in-memory but also to persist the cache which leads to a much improved
      99             :         // cold-start behavior and in-memory miss latency.
     100             : 
     101         152 :         cache(BYUUID_NAME, String.class, new TypeLiteral<Optional<InternalGroup>>() {})
     102         152 :             .maximumWeight(Long.MAX_VALUE)
     103         152 :             .loader(ByUUIDInMemoryLoader.class);
     104             : 
     105         152 :         persist(
     106             :                 BYUUID_NAME_PERSISTED,
     107             :                 Cache.GroupKeyProto.class,
     108         152 :                 new TypeLiteral<InternalGroup>() {})
     109         152 :             .loader(PersistedByUUIDLoader.class)
     110         152 :             .keySerializer(new ProtobufSerializer<>(Cache.GroupKeyProto.parser()))
     111         152 :             .valueSerializer(PersistedInternalGroupSerializer.INSTANCE)
     112         152 :             .diskLimit(1 << 30) // 1 GiB
     113         152 :             .version(1)
     114         152 :             .maximumWeight(0);
     115             : 
     116         152 :         bind(GroupCacheImpl.class);
     117         152 :         bind(GroupCache.class).to(GroupCacheImpl.class);
     118         152 :       }
     119             :     };
     120             :   }
     121             : 
     122             :   private final LoadingCache<AccountGroup.Id, Optional<InternalGroup>> byId;
     123             :   private final LoadingCache<String, Optional<InternalGroup>> byName;
     124             :   private final LoadingCache<String, Optional<InternalGroup>> byUUID;
     125             : 
     126             :   @Inject
     127             :   GroupCacheImpl(
     128             :       @Named(BYID_NAME) LoadingCache<AccountGroup.Id, Optional<InternalGroup>> byId,
     129             :       @Named(BYNAME_NAME) LoadingCache<String, Optional<InternalGroup>> byName,
     130         151 :       @Named(BYUUID_NAME) LoadingCache<String, Optional<InternalGroup>> byUUID) {
     131         151 :     this.byId = byId;
     132         151 :     this.byName = byName;
     133         151 :     this.byUUID = byUUID;
     134         151 :   }
     135             : 
     136             :   @Override
     137             :   public Optional<InternalGroup> get(AccountGroup.Id groupId) {
     138             :     try {
     139           7 :       return byId.get(groupId);
     140           0 :     } catch (ExecutionException e) {
     141           0 :       logger.atWarning().withCause(e).log("Cannot load group %s", groupId);
     142           0 :       return Optional.empty();
     143             :     }
     144             :   }
     145             : 
     146             :   @Override
     147             :   public Optional<InternalGroup> get(AccountGroup.NameKey name) {
     148         150 :     if (name == null) {
     149           0 :       return Optional.empty();
     150             :     }
     151             :     try {
     152         150 :       return byName.get(name.get());
     153           0 :     } catch (ExecutionException e) {
     154           0 :       logger.atWarning().withCause(e).log("Cannot look up group %s by name", name.get());
     155           0 :       return Optional.empty();
     156             :     }
     157             :   }
     158             : 
     159             :   @Override
     160             :   public Optional<InternalGroup> get(AccountGroup.UUID groupUuid) {
     161         151 :     if (groupUuid == null) {
     162           0 :       return Optional.empty();
     163             :     }
     164             : 
     165             :     try {
     166         151 :       return byUUID.get(groupUuid.get());
     167           0 :     } catch (ExecutionException e) {
     168           0 :       logger.atWarning().withCause(e).log("Cannot look up group %s by uuid", groupUuid.get());
     169           0 :       return Optional.empty();
     170             :     }
     171             :   }
     172             : 
     173             :   @Override
     174             :   public Map<AccountGroup.UUID, InternalGroup> get(Collection<AccountGroup.UUID> groupUuids) {
     175             :     try {
     176         150 :       Set<String> groupUuidsStringSet =
     177         150 :           groupUuids.stream().map(u -> u.get()).collect(toImmutableSet());
     178         150 :       return byUUID.getAll(groupUuidsStringSet).entrySet().stream()
     179         150 :           .filter(g -> g.getValue().isPresent())
     180         150 :           .collect(toImmutableMap(g -> AccountGroup.uuid(g.getKey()), g -> g.getValue().get()));
     181           0 :     } catch (ExecutionException e) {
     182           0 :       logger.atWarning().withCause(e).log("Cannot look up groups %s by uuids", groupUuids);
     183           0 :       return ImmutableMap.of();
     184             :     }
     185             :   }
     186             : 
     187             :   @Override
     188             :   public void evict(AccountGroup.Id groupId) {
     189         151 :     if (groupId != null) {
     190         151 :       logger.atFine().log("Evict group %s by ID", groupId.get());
     191         151 :       byId.invalidate(groupId);
     192             :     }
     193         151 :   }
     194             : 
     195             :   @Override
     196             :   public void evict(AccountGroup.NameKey groupName) {
     197         151 :     if (groupName != null) {
     198         151 :       logger.atFine().log("Evict group '%s' by name", groupName.get());
     199         151 :       byName.invalidate(groupName.get());
     200             :     }
     201         151 :   }
     202             : 
     203             :   @Override
     204             :   public void evict(AccountGroup.UUID groupUuid) {
     205         151 :     if (groupUuid != null) {
     206         151 :       logger.atFine().log("Evict group %s by UUID", groupUuid.get());
     207         151 :       byUUID.invalidate(groupUuid.get());
     208             :     }
     209         151 :   }
     210             : 
     211             :   @Override
     212             :   public void evict(Collection<AccountGroup.UUID> groupUuids) {
     213          15 :     if (groupUuids != null && !groupUuids.isEmpty()) {
     214          15 :       logger.atFine().log("Evict groups %s by UUID", groupUuids);
     215          15 :       byUUID.invalidateAll(groupUuids);
     216             :     }
     217          15 :   }
     218             : 
     219             :   static class ByIdLoader extends CacheLoader<AccountGroup.Id, Optional<InternalGroup>> {
     220             :     private final Provider<InternalGroupQuery> groupQueryProvider;
     221             : 
     222             :     @Inject
     223         152 :     ByIdLoader(Provider<InternalGroupQuery> groupQueryProvider) {
     224         152 :       this.groupQueryProvider = groupQueryProvider;
     225         152 :     }
     226             : 
     227             :     @Override
     228             :     public Optional<InternalGroup> load(AccountGroup.Id key) throws Exception {
     229           7 :       try (TraceTimer ignored =
     230           7 :           TraceContext.newTimer(
     231           7 :               "Loading group by ID", Metadata.builder().groupId(key.get()).build())) {
     232           7 :         return groupQueryProvider.get().byId(key);
     233             :       }
     234             :     }
     235             :   }
     236             : 
     237             :   static class ByNameLoader extends CacheLoader<String, Optional<InternalGroup>> {
     238             :     private final Provider<InternalGroupQuery> groupQueryProvider;
     239             : 
     240             :     @Inject
     241         152 :     ByNameLoader(Provider<InternalGroupQuery> groupQueryProvider) {
     242         152 :       this.groupQueryProvider = groupQueryProvider;
     243         152 :     }
     244             : 
     245             :     @Override
     246             :     public Optional<InternalGroup> load(String name) throws Exception {
     247         150 :       try (TraceTimer ignored =
     248         150 :           TraceContext.newTimer(
     249         150 :               "Loading group by name", Metadata.builder().groupName(name).build())) {
     250         150 :         return groupQueryProvider.get().byName(AccountGroup.nameKey(name));
     251             :       }
     252             :     }
     253             :   }
     254             : 
     255             :   static class ByUUIDInMemoryLoader extends CacheLoader<String, Optional<InternalGroup>> {
     256             :     private final LoadingCache<Cache.GroupKeyProto, InternalGroup> persistedCache;
     257             :     private final GitRepositoryManager repoManager;
     258             :     private final AllUsersName allUsersName;
     259             : 
     260             :     @Inject
     261             :     ByUUIDInMemoryLoader(
     262             :         @Named(BYUUID_NAME_PERSISTED)
     263             :             LoadingCache<Cache.GroupKeyProto, InternalGroup> persistedCache,
     264             :         GitRepositoryManager repoManager,
     265         152 :         AllUsersName allUsersName) {
     266         152 :       this.persistedCache = persistedCache;
     267         152 :       this.repoManager = repoManager;
     268         152 :       this.allUsersName = allUsersName;
     269         152 :     }
     270             : 
     271             :     @Override
     272             :     public Optional<InternalGroup> load(String uuid) throws Exception {
     273         151 :       return loadAll(ImmutableSet.of(uuid)).get(uuid);
     274             :     }
     275             : 
     276             :     @Override
     277             :     public Map<String, Optional<InternalGroup>> loadAll(Iterable<? extends String> uuids)
     278             :         throws Exception {
     279         151 :       Map<String, Optional<InternalGroup>> toReturn = new HashMap<>();
     280         151 :       if (Iterables.isEmpty(uuids)) {
     281           0 :         return toReturn;
     282             :       }
     283         151 :       Iterator<? extends String> uuidIterator = uuids.iterator();
     284         151 :       List<Cache.GroupKeyProto> keyList = new ArrayList<>();
     285         151 :       try (TraceTimer ignored =
     286         151 :               TraceContext.newTimer(
     287             :                   "Loading group from serialized cache",
     288         151 :                   Metadata.builder().cacheName(BYUUID_NAME_PERSISTED).build());
     289         151 :           Repository allUsers = repoManager.openRepository(allUsersName)) {
     290         151 :         while (uuidIterator.hasNext()) {
     291         151 :           String currentUuid = uuidIterator.next();
     292         151 :           String ref = RefNames.refsGroups(AccountGroup.uuid(currentUuid));
     293         151 :           Ref sha1 = allUsers.exactRef(ref);
     294         151 :           if (sha1 == null) {
     295          30 :             toReturn.put(currentUuid, Optional.empty());
     296          30 :             continue;
     297             :           }
     298             :           Cache.GroupKeyProto key =
     299         151 :               Cache.GroupKeyProto.newBuilder()
     300         151 :                   .setUuid(currentUuid)
     301         151 :                   .setRevision(ObjectIdConverter.create().toByteString(sha1.getObjectId()))
     302         151 :                   .build();
     303         151 :           keyList.add(key);
     304         151 :         }
     305             :       }
     306         151 :       persistedCache.getAll(keyList).entrySet().stream()
     307         151 :           .forEach(g -> toReturn.put(g.getKey().getUuid(), Optional.of(g.getValue())));
     308         151 :       return toReturn;
     309             :     }
     310             :   }
     311             : 
     312             :   static class PersistedByUUIDLoader extends CacheLoader<Cache.GroupKeyProto, InternalGroup> {
     313             :     private final Groups groups;
     314             : 
     315             :     @Inject
     316         152 :     PersistedByUUIDLoader(Groups groups) {
     317         152 :       this.groups = groups;
     318         152 :     }
     319             : 
     320             :     @Override
     321             :     public InternalGroup load(Cache.GroupKeyProto key) throws Exception {
     322         151 :       try (TraceTimer ignored =
     323         151 :           TraceContext.newTimer(
     324         151 :               "Loading group by UUID", Metadata.builder().groupUuid(key.getUuid()).build())) {
     325         151 :         ObjectId sha1 = ObjectIdConverter.create().fromByteString(key.getRevision());
     326         151 :         Optional<InternalGroup> loadedGroup =
     327         151 :             groups.getGroup(AccountGroup.uuid(key.getUuid()), sha1);
     328         151 :         if (!loadedGroup.isPresent()) {
     329           0 :           throw new IllegalStateException(
     330           0 :               String.format(
     331             :                   "group %s should have the sha-1 %s, but " + "it was not found",
     332           0 :                   key.getUuid(), sha1.getName()));
     333             :         }
     334         151 :         return loadedGroup.get();
     335             :       }
     336             :     }
     337             :   }
     338             : 
     339         152 :   private enum PersistedInternalGroupSerializer implements CacheSerializer<InternalGroup> {
     340         152 :     INSTANCE;
     341             : 
     342             :     @Override
     343             :     public byte[] serialize(InternalGroup value) {
     344          15 :       if (value == null) {
     345           0 :         return new byte[0];
     346             :       }
     347          15 :       return Protos.toByteArray(InternalGroupSerializer.serialize(value));
     348             :     }
     349             : 
     350             :     @Nullable
     351             :     @Override
     352             :     public InternalGroup deserialize(byte[] in) {
     353          15 :       if (Strings.fromByteArray(in).isEmpty()) {
     354           0 :         return null;
     355             :       }
     356          15 :       return InternalGroupSerializer.deserialize(
     357          15 :           Protos.parseUnchecked(Cache.InternalGroupProto.parser(), in));
     358             :     }
     359             :   }
     360             : }