Line data Source code
1 : // Copyright (C) 2008 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.project;
16 :
17 : import static com.google.gerrit.server.project.ProjectCache.illegalState;
18 : import static java.nio.charset.StandardCharsets.UTF_8;
19 : import static java.util.stream.Collectors.toSet;
20 :
21 : import com.google.common.annotations.VisibleForTesting;
22 : import com.google.common.cache.CacheLoader;
23 : import com.google.common.cache.LoadingCache;
24 : import com.google.common.collect.ImmutableSet;
25 : import com.google.common.collect.ImmutableSortedSet;
26 : import com.google.common.collect.Sets;
27 : import com.google.common.collect.Streams;
28 : import com.google.common.flogger.FluentLogger;
29 : import com.google.common.hash.Hashing;
30 : import com.google.common.util.concurrent.Futures;
31 : import com.google.common.util.concurrent.ListenableFuture;
32 : import com.google.common.util.concurrent.ListeningExecutorService;
33 : import com.google.gerrit.common.Nullable;
34 : import com.google.gerrit.entities.AccountGroup;
35 : import com.google.gerrit.entities.CachedProjectConfig;
36 : import com.google.gerrit.entities.Project;
37 : import com.google.gerrit.entities.RefNames;
38 : import com.google.gerrit.exceptions.StorageException;
39 : import com.google.gerrit.index.project.ProjectIndexer;
40 : import com.google.gerrit.lifecycle.LifecycleModule;
41 : import com.google.gerrit.metrics.Counter2;
42 : import com.google.gerrit.metrics.Description;
43 : import com.google.gerrit.metrics.Description.Units;
44 : import com.google.gerrit.metrics.Field;
45 : import com.google.gerrit.metrics.MetricMaker;
46 : import com.google.gerrit.metrics.Timer0;
47 : import com.google.gerrit.proto.Protos;
48 : import com.google.gerrit.server.CacheRefreshExecutor;
49 : import com.google.gerrit.server.cache.CacheModule;
50 : import com.google.gerrit.server.cache.proto.Cache;
51 : import com.google.gerrit.server.cache.serialize.CacheSerializer;
52 : import com.google.gerrit.server.cache.serialize.ObjectIdConverter;
53 : import com.google.gerrit.server.cache.serialize.ProtobufSerializer;
54 : import com.google.gerrit.server.cache.serialize.entities.CachedProjectConfigSerializer;
55 : import com.google.gerrit.server.config.AllProjectsConfigProvider;
56 : import com.google.gerrit.server.config.AllProjectsName;
57 : import com.google.gerrit.server.config.AllUsersName;
58 : import com.google.gerrit.server.config.GerritServerConfig;
59 : import com.google.gerrit.server.git.GitRepositoryManager;
60 : import com.google.gerrit.server.logging.Metadata;
61 : import com.google.gerrit.server.logging.TraceContext;
62 : import com.google.gerrit.server.logging.TraceContext.TraceTimer;
63 : import com.google.inject.Inject;
64 : import com.google.inject.Module;
65 : import com.google.inject.Provider;
66 : import com.google.inject.Singleton;
67 : import com.google.inject.TypeLiteral;
68 : import com.google.inject.name.Named;
69 : import com.google.protobuf.ByteString;
70 : import java.io.IOException;
71 : import java.time.Duration;
72 : import java.util.Arrays;
73 : import java.util.Objects;
74 : import java.util.Optional;
75 : import java.util.Set;
76 : import java.util.concurrent.ExecutionException;
77 : import java.util.concurrent.locks.Lock;
78 : import java.util.concurrent.locks.ReentrantLock;
79 : import org.eclipse.jgit.errors.ConfigInvalidException;
80 : import org.eclipse.jgit.errors.RepositoryNotFoundException;
81 : import org.eclipse.jgit.lib.Config;
82 : import org.eclipse.jgit.lib.ObjectId;
83 : import org.eclipse.jgit.lib.Ref;
84 : import org.eclipse.jgit.lib.Repository;
85 : import org.eclipse.jgit.lib.StoredConfig;
86 :
87 : /**
88 : * Cache of project information, including access rights.
89 : *
90 : * <p>The data of a project is the project's project.config in refs/meta/config parsed out as an
91 : * immutable value. It's keyed purely by the refs/meta/config SHA-1. We also cache the same value
92 : * keyed by name. The latter mapping can become outdated, so data must be evicted explicitly.
93 : */
94 : @Singleton
95 : public class ProjectCacheImpl implements ProjectCache {
96 152 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
97 :
98 : public static final String CACHE_NAME = "projects";
99 :
100 : public static final String PERSISTED_CACHE_NAME = "persisted_projects";
101 :
102 : public static final String CACHE_LIST = "project_list";
103 :
104 : public static Module module() {
105 152 : return new CacheModule() {
106 : @Override
107 : protected void configure() {
108 : // We split the project cache into two parts for performance reasons:
109 : // 1) An in-memory part that has only the project name as key.
110 : // 2) A persisted part that has the name and revision as key.
111 : //
112 : // When loading dashboards or returning change query results we potentially
113 : // need to access hundreds of projects because each change could originate in
114 : // a different project and, through inheritance, require us to check even more
115 : // projects when evaluating permissions. It's not feasible to read the revision
116 : // of refs/meta/config from each of these repos as that would require opening
117 : // them all and reading their ref list or table.
118 : // At the same time, we want the persisted cache to be immutable and we want it
119 : // to be impossible that a value for a given key is out of date. We therefore
120 : // require a revision in the key. That is in line with the rest of the caches in
121 : // Gerrit.
122 : //
123 : // Splitting the cache into two chunks internally in this class allows us to retain
124 : // the existing performance guarantees of not requiring reads for the repo for values
125 : // cached in-memory but also to persist the cache which leads to a much improved
126 : // cold-start behavior and in-memory miss latency.
127 152 : cache(CACHE_NAME, Project.NameKey.class, CachedProjectConfig.class)
128 152 : .loader(InMemoryLoader.class)
129 152 : .refreshAfterWrite(Duration.ofMinutes(15))
130 152 : .expireAfterWrite(Duration.ofHours(1));
131 :
132 152 : persist(PERSISTED_CACHE_NAME, Cache.ProjectCacheKeyProto.class, CachedProjectConfig.class)
133 152 : .loader(PersistedLoader.class)
134 152 : .keySerializer(new ProtobufSerializer<>(Cache.ProjectCacheKeyProto.parser()))
135 152 : .valueSerializer(PersistedProjectConfigSerializer.INSTANCE)
136 152 : .diskLimit(1 << 30) // 1 GiB
137 152 : .version(4)
138 152 : .maximumWeight(0);
139 :
140 152 : cache(CACHE_LIST, ListKey.class, new TypeLiteral<ImmutableSortedSet<Project.NameKey>>() {})
141 152 : .maximumWeight(1)
142 152 : .loader(Lister.class);
143 :
144 152 : bind(ProjectCacheImpl.class);
145 152 : bind(ProjectCache.class).to(ProjectCacheImpl.class);
146 :
147 152 : install(
148 152 : new LifecycleModule() {
149 : @Override
150 : protected void configure() {
151 152 : listener().to(ProjectCacheWarmer.class);
152 152 : }
153 : });
154 152 : install(
155 152 : new LifecycleModule() {
156 : @Override
157 : protected void configure() {
158 152 : listener().to(PeriodicProjectListCacheWarmer.LifeCycle.class);
159 152 : }
160 : });
161 152 : }
162 : };
163 : }
164 :
165 : private final Config config;
166 : private final AllProjectsName allProjectsName;
167 : private final AllUsersName allUsersName;
168 : private final LoadingCache<Project.NameKey, CachedProjectConfig> inMemoryProjectCache;
169 : private final LoadingCache<ListKey, ImmutableSortedSet<Project.NameKey>> list;
170 : private final Lock listLock;
171 : private final Provider<ProjectIndexer> indexer;
172 : private final Timer0 guessRelevantGroupsLatency;
173 : private final ProjectState.Factory projectStateFactory;
174 :
175 : @Inject
176 : ProjectCacheImpl(
177 : @GerritServerConfig Config config,
178 : AllProjectsName allProjectsName,
179 : AllUsersName allUsersName,
180 : @Named(CACHE_NAME) LoadingCache<Project.NameKey, CachedProjectConfig> inMemoryProjectCache,
181 : @Named(CACHE_LIST) LoadingCache<ListKey, ImmutableSortedSet<Project.NameKey>> list,
182 : Provider<ProjectIndexer> indexer,
183 : MetricMaker metricMaker,
184 152 : ProjectState.Factory projectStateFactory) {
185 152 : this.config = config;
186 152 : this.allProjectsName = allProjectsName;
187 152 : this.allUsersName = allUsersName;
188 152 : this.inMemoryProjectCache = inMemoryProjectCache;
189 152 : this.list = list;
190 152 : this.listLock = new ReentrantLock(true /* fair */);
191 152 : this.indexer = indexer;
192 152 : this.projectStateFactory = projectStateFactory;
193 :
194 152 : this.guessRelevantGroupsLatency =
195 152 : metricMaker.newTimer(
196 : "group/guess_relevant_groups_latency",
197 : new Description("Latency for guessing relevant groups")
198 152 : .setCumulative()
199 152 : .setUnit(Units.NANOSECONDS));
200 152 : }
201 :
202 : @Override
203 : public ProjectState getAllProjects() {
204 151 : return get(allProjectsName).orElseThrow(illegalState(allProjectsName));
205 : }
206 :
207 : @Override
208 : public ProjectState getAllUsers() {
209 16 : return get(allUsersName).orElseThrow(illegalState(allUsersName));
210 : }
211 :
212 : @Override
213 : public Optional<ProjectState> get(@Nullable Project.NameKey projectName) {
214 151 : if (projectName == null) {
215 0 : return Optional.empty();
216 : }
217 :
218 : try {
219 151 : return Optional.of(inMemoryProjectCache.get(projectName)).map(projectStateFactory::create);
220 143 : } catch (ExecutionException e) {
221 143 : if ((e.getCause() instanceof RepositoryNotFoundException)) {
222 143 : logger.atFine().log("Cannot find project %s", projectName.get());
223 143 : return Optional.empty();
224 : }
225 2 : throw new StorageException(
226 2 : String.format("project state of project %s not available", projectName.get()), e);
227 : }
228 : }
229 :
230 : @Override
231 : public void evict(Project.NameKey p) {
232 100 : if (p != null) {
233 100 : logger.atFine().log("Evict project '%s'", p.get());
234 100 : inMemoryProjectCache.invalidate(p);
235 : }
236 100 : }
237 :
238 : @Override
239 : public void evictAndReindex(Project p) {
240 61 : evictAndReindex(p.getNameKey());
241 61 : }
242 :
243 : @Override
244 : public void evictAndReindex(Project.NameKey p) {
245 92 : evict(p);
246 92 : indexer.get().index(p);
247 92 : }
248 :
249 : @Override
250 : public void remove(Project p) {
251 0 : remove(p.getNameKey());
252 0 : }
253 :
254 : @Override
255 : public void remove(Project.NameKey name) {
256 0 : listLock.lock();
257 : try {
258 0 : list.put(
259 : ListKey.ALL,
260 0 : ImmutableSortedSet.copyOf(Sets.difference(list.get(ListKey.ALL), ImmutableSet.of(name))));
261 0 : } catch (ExecutionException e) {
262 0 : logger.atWarning().withCause(e).log("Cannot list available projects");
263 : } finally {
264 0 : listLock.unlock();
265 : }
266 0 : evictAndReindex(name);
267 0 : }
268 :
269 : @Override
270 : public void onCreateProject(Project.NameKey newProjectName) throws IOException {
271 144 : listLock.lock();
272 : try {
273 144 : list.put(
274 : ListKey.ALL,
275 144 : ImmutableSortedSet.copyOf(
276 144 : Sets.union(list.get(ListKey.ALL), ImmutableSet.of(newProjectName))));
277 0 : } catch (ExecutionException e) {
278 0 : logger.atWarning().withCause(e).log("Cannot list available projects");
279 : } finally {
280 144 : listLock.unlock();
281 : }
282 144 : indexer.get().index(newProjectName);
283 144 : }
284 :
285 : @Override
286 : public ImmutableSortedSet<Project.NameKey> all() {
287 : try {
288 34 : return list.get(ListKey.ALL);
289 0 : } catch (ExecutionException e) {
290 0 : logger.atWarning().withCause(e).log("Cannot list available projects");
291 0 : return ImmutableSortedSet.of();
292 : }
293 : }
294 :
295 : @Override
296 : public void refreshProjectList() {
297 0 : list.refresh(ListKey.ALL);
298 0 : }
299 :
300 : @Override
301 : public Set<AccountGroup.UUID> guessRelevantGroupUUIDs() {
302 1 : try (Timer0.Context ignored = guessRelevantGroupsLatency.start()) {
303 1 : return Streams.concat(
304 1 : Arrays.stream(config.getStringList("groups", /* subsection= */ null, "relevantGroup"))
305 1 : .map(AccountGroup::uuid),
306 1 : all().stream()
307 1 : .map(n -> inMemoryProjectCache.getIfPresent(n))
308 1 : .filter(Objects::nonNull)
309 1 : .flatMap(p -> p.getAllGroupUUIDs().stream())
310 : // getAllGroupUUIDs shouldn't really return null UUIDs, but harden
311 : // against them just in case there is a bug or corner case.
312 1 : .filter(id -> id != null && id.get() != null))
313 1 : .collect(toSet());
314 : }
315 : }
316 :
317 : @Override
318 : public ImmutableSortedSet<Project.NameKey> byName(String pfx) {
319 1 : Project.NameKey start = Project.nameKey(pfx);
320 1 : Project.NameKey end = Project.nameKey(pfx + Character.MAX_VALUE);
321 : try {
322 : // Right endpoint is exclusive, but U+FFFF is a non-character so no project ends with it.
323 1 : return list.get(ListKey.ALL).subSet(start, end);
324 0 : } catch (ExecutionException e) {
325 0 : logger.atWarning().withCause(e).log("Cannot look up projects for prefix %s", pfx);
326 0 : return ImmutableSortedSet.of();
327 : }
328 : }
329 :
330 : /**
331 : * Returns a {@code MurMur128} hash of the contents of {@code etc/All-Projects-project.config}.
332 : */
333 : public static byte[] allProjectsFileProjectConfigHash(Optional<StoredConfig> allProjectsConfig) {
334 : // Hash the contents of All-Projects-project.config
335 : // This is a way for administrators to orchestrate project.config changes across many Gerrit
336 : // instances.
337 : // When this file changes, we need to make sure we disregard persistently cached project
338 : // state.
339 151 : if (!allProjectsConfig.isPresent()) {
340 : // If the project.config file is not present, this is equal to an empty config file:
341 0 : return Hashing.murmur3_128().hashString("", UTF_8).asBytes();
342 : }
343 : try {
344 151 : allProjectsConfig.get().load();
345 0 : } catch (IOException | ConfigInvalidException e) {
346 0 : throw new IllegalStateException(e);
347 151 : }
348 151 : return Hashing.murmur3_128().hashString(allProjectsConfig.get().toText(), UTF_8).asBytes();
349 : }
350 :
351 : @Singleton
352 : static class InMemoryLoader extends CacheLoader<Project.NameKey, CachedProjectConfig> {
353 : private final LoadingCache<Cache.ProjectCacheKeyProto, CachedProjectConfig> persistedCache;
354 : private final GitRepositoryManager repoManager;
355 : private final ListeningExecutorService cacheRefreshExecutor;
356 : private final Counter2<String, Boolean> refreshCounter;
357 : private final AllProjectsName allProjectsName;
358 : private final AllProjectsConfigProvider allProjectsConfigProvider;
359 :
360 : @Inject
361 : InMemoryLoader(
362 : @Named(PERSISTED_CACHE_NAME)
363 : LoadingCache<Cache.ProjectCacheKeyProto, CachedProjectConfig> persistedCache,
364 : GitRepositoryManager repoManager,
365 : @CacheRefreshExecutor ListeningExecutorService cacheRefreshExecutor,
366 : MetricMaker metricMaker,
367 : AllProjectsName allProjectsName,
368 152 : AllProjectsConfigProvider allProjectsConfigProvider) {
369 152 : this.persistedCache = persistedCache;
370 152 : this.repoManager = repoManager;
371 152 : this.cacheRefreshExecutor = cacheRefreshExecutor;
372 152 : refreshCounter =
373 152 : metricMaker.newCounter(
374 : "caches/refresh_count",
375 : new Description(
376 : "The number of refreshes per cache with an indicator if a reload was"
377 : + " necessary.")
378 152 : .setRate(),
379 152 : Field.ofString("cache", Metadata.Builder::className)
380 152 : .description("The name of the cache.")
381 152 : .build(),
382 152 : Field.ofBoolean("outdated", Metadata.Builder::outdated)
383 152 : .description("Whether the cache entry was outdated on reload.")
384 152 : .build());
385 152 : this.allProjectsName = allProjectsName;
386 152 : this.allProjectsConfigProvider = allProjectsConfigProvider;
387 152 : }
388 :
389 : @Override
390 : public CachedProjectConfig load(Project.NameKey key) throws IOException, ExecutionException {
391 151 : try (TraceTimer ignored =
392 151 : TraceContext.newTimer(
393 : "Loading project from serialized cache",
394 151 : Metadata.builder().projectName(key.get()).build());
395 151 : Repository git = repoManager.openRepository(key)) {
396 : Cache.ProjectCacheKeyProto.Builder keyProto =
397 151 : Cache.ProjectCacheKeyProto.newBuilder().setProject(key.get());
398 151 : Ref configRef = git.exactRef(RefNames.REFS_CONFIG);
399 151 : if (key.get().equals(allProjectsName.get())) {
400 151 : Optional<StoredConfig> allProjectsConfig = allProjectsConfigProvider.get(allProjectsName);
401 151 : byte[] fileHash = allProjectsFileProjectConfigHash(allProjectsConfig);
402 151 : keyProto.setGlobalConfigRevision(ByteString.copyFrom(fileHash));
403 : }
404 151 : if (configRef != null) {
405 151 : keyProto.setRevision(ObjectIdConverter.create().toByteString(configRef.getObjectId()));
406 : }
407 151 : return persistedCache.get(keyProto.build());
408 : }
409 : }
410 :
411 : @Override
412 : public ListenableFuture<CachedProjectConfig> reload(
413 : Project.NameKey key, CachedProjectConfig oldState) throws Exception {
414 0 : try (TraceTimer ignored =
415 0 : TraceContext.newTimer(
416 0 : "Reload project", Metadata.builder().projectName(key.get()).build())) {
417 0 : try (Repository git = repoManager.openRepository(key)) {
418 0 : Ref configRef = git.exactRef(RefNames.REFS_CONFIG);
419 0 : if (configRef != null && configRef.getObjectId().equals(oldState.getRevision().get())) {
420 0 : refreshCounter.increment(CACHE_NAME, false);
421 0 : return Futures.immediateFuture(oldState);
422 : }
423 0 : }
424 :
425 : // Repository is not thread safe, so we have to open it on the thread that does the loading.
426 : // Just invoke the loader on the other thread.
427 0 : refreshCounter.increment(CACHE_NAME, true);
428 0 : return cacheRefreshExecutor.submit(() -> load(key));
429 0 : }
430 : }
431 : }
432 :
433 : @Singleton
434 : static class PersistedLoader
435 : extends CacheLoader<Cache.ProjectCacheKeyProto, CachedProjectConfig> {
436 : private final GitRepositoryManager repoManager;
437 : private final ProjectConfig.Factory projectConfigFactory;
438 :
439 : @Inject
440 152 : PersistedLoader(GitRepositoryManager repoManager, ProjectConfig.Factory projectConfigFactory) {
441 152 : this.repoManager = repoManager;
442 152 : this.projectConfigFactory = projectConfigFactory;
443 152 : }
444 :
445 : @Override
446 : public CachedProjectConfig load(Cache.ProjectCacheKeyProto key) throws Exception {
447 151 : Project.NameKey nameKey = Project.nameKey(key.getProject());
448 : ObjectId revision =
449 151 : key.getRevision().isEmpty()
450 3 : ? null
451 151 : : ObjectIdConverter.create().fromByteString(key.getRevision());
452 151 : try (TraceTimer ignored =
453 151 : TraceContext.newTimer(
454 151 : "Loading project from repo", Metadata.builder().projectName(nameKey.get()).build())) {
455 151 : try (Repository git = repoManager.openRepository(nameKey)) {
456 151 : ProjectConfig cfg = projectConfigFactory.create(nameKey);
457 151 : cfg.load(git, revision);
458 151 : return cfg.getCacheable();
459 : }
460 : }
461 : }
462 : }
463 :
464 152 : private enum PersistedProjectConfigSerializer implements CacheSerializer<CachedProjectConfig> {
465 152 : INSTANCE;
466 :
467 : @Override
468 : public byte[] serialize(CachedProjectConfig value) {
469 15 : return Protos.toByteArray(CachedProjectConfigSerializer.serialize(value));
470 : }
471 :
472 : @Override
473 : public CachedProjectConfig deserialize(byte[] in) {
474 15 : return CachedProjectConfigSerializer.deserialize(
475 15 : Protos.parseUnchecked(Cache.CachedProjectConfigProto.parser(), in));
476 : }
477 : }
478 :
479 : static class ListKey {
480 147 : static final ListKey ALL = new ListKey();
481 :
482 : private ListKey() {}
483 : }
484 :
485 : static class Lister extends CacheLoader<ListKey, ImmutableSortedSet<Project.NameKey>> {
486 : private final GitRepositoryManager mgr;
487 :
488 : @Inject
489 152 : Lister(GitRepositoryManager mgr) {
490 152 : this.mgr = mgr;
491 152 : }
492 :
493 : @Override
494 : public ImmutableSortedSet<Project.NameKey> load(ListKey key) throws Exception {
495 147 : try (TraceTimer timer = TraceContext.newTimer("Loading project list")) {
496 147 : return ImmutableSortedSet.copyOf(mgr.list());
497 : }
498 : }
499 : }
500 :
501 : @VisibleForTesting
502 : public void evictAllByName() {
503 1 : inMemoryProjectCache.invalidateAll();
504 1 : }
505 :
506 : @VisibleForTesting
507 : public long sizeAllByName() {
508 1 : return inMemoryProjectCache.size();
509 : }
510 : }
|
