Line data Source code
1 : // Copyright (C) 2017 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.acceptance;
16 :
17 : import static com.google.common.base.Preconditions.checkState;
18 : import static com.google.gerrit.entities.RefNames.REFS_USERS;
19 : import static java.util.stream.Collectors.toSet;
20 :
21 : import com.google.common.collect.ImmutableList;
22 : import com.google.common.collect.Multimap;
23 : import com.google.common.collect.MultimapBuilder;
24 : import com.google.common.collect.Sets;
25 : import com.google.gerrit.common.Nullable;
26 : import com.google.gerrit.entities.Account;
27 : import com.google.gerrit.entities.AccountGroup;
28 : import com.google.gerrit.entities.Project;
29 : import com.google.gerrit.entities.RefNames;
30 : import com.google.gerrit.index.RefState;
31 : import com.google.gerrit.server.account.AccountCache;
32 : import com.google.gerrit.server.account.GroupCache;
33 : import com.google.gerrit.server.account.GroupIncludeCache;
34 : import com.google.gerrit.server.config.AllUsersName;
35 : import com.google.gerrit.server.git.GitRepositoryManager;
36 : import com.google.gerrit.server.index.account.AccountIndexer;
37 : import com.google.gerrit.server.index.group.GroupIndexer;
38 : import com.google.gerrit.server.project.ProjectCache;
39 : import com.google.gerrit.server.project.RefPatternMatcher;
40 : import com.google.inject.Inject;
41 : import java.io.IOException;
42 : import java.util.Arrays;
43 : import java.util.Collection;
44 : import java.util.HashSet;
45 : import java.util.List;
46 : import java.util.Map;
47 : import java.util.Objects;
48 : import java.util.Set;
49 : import java.util.stream.Stream;
50 : import org.eclipse.jgit.lib.ObjectId;
51 : import org.eclipse.jgit.lib.Ref;
52 : import org.eclipse.jgit.lib.RefUpdate;
53 : import org.eclipse.jgit.lib.Repository;
54 :
55 : /**
56 : * Saves the states of given projects and resets the project states on close.
57 : *
58 : * <p>Saving the project states is done by saving the states of all refs in the project. On close
59 : * those refs are reset to the saved states. Refs that were newly created are deleted.
60 : *
61 : * <p>By providing ref patterns per project it can be controlled which refs should be reset on
62 : * close.
63 : *
64 : * <p>If resetting touches {@code refs/meta/config} branches the corresponding projects are evicted
65 : * from the project cache.
66 : *
67 : * <p>If resetting touches user branches or the {@code refs/meta/external-ids} branch the
68 : * corresponding accounts are evicted from the account cache and also if needed from the cache in
69 : * {@link AccountCreator}.
70 : *
71 : * <p>At the moment this class has the following limitations:
72 : *
73 : * <ul>
74 : * <li>Resetting group branches doesn't evict the corresponding groups from the group cache.
75 : * <li>Changes are not reindexed if change meta refs are reset.
76 : * <li>Changes are not reindexed if starred-changes refs in All-Users are reset.
77 : * <li>If accounts are deleted changes may still refer to these accounts (e.g. as reviewers).
78 : * </ul>
79 : *
80 : * Primarily this class is intended to reset the states of the All-Projects and All-Users projects
81 : * after each test. These projects rarely contain changes and it's currently not a problem if these
82 : * changes get stale. For creating changes each test gets a brand new project. Since this project is
83 : * not used outside of the test method that creates it, it doesn't need to be reset.
84 : */
85 : public class ProjectResetter implements AutoCloseable {
86 : public static class Builder {
87 : public interface Factory {
88 : Builder builder();
89 : }
90 :
91 : private final GitRepositoryManager repoManager;
92 : private final AllUsersName allUsersName;
93 : @Nullable private final AccountCreator accountCreator;
94 : @Nullable private final AccountCache accountCache;
95 : @Nullable private final AccountIndexer accountIndexer;
96 : @Nullable private final GroupCache groupCache;
97 : @Nullable private final GroupIncludeCache groupIncludeCache;
98 : @Nullable private final GroupIndexer groupIndexer;
99 : @Nullable private final ProjectCache projectCache;
100 :
101 : @Inject
102 : public Builder(
103 : GitRepositoryManager repoManager,
104 : AllUsersName allUsersName,
105 : @Nullable AccountCreator accountCreator,
106 : @Nullable AccountCache accountCache,
107 : @Nullable AccountIndexer accountIndexer,
108 : @Nullable GroupCache groupCache,
109 : @Nullable GroupIncludeCache groupIncludeCache,
110 : @Nullable GroupIndexer groupIndexer,
111 132 : @Nullable ProjectCache projectCache) {
112 132 : this.repoManager = repoManager;
113 132 : this.allUsersName = allUsersName;
114 132 : this.accountCreator = accountCreator;
115 132 : this.accountCache = accountCache;
116 132 : this.accountIndexer = accountIndexer;
117 132 : this.groupCache = groupCache;
118 132 : this.groupIncludeCache = groupIncludeCache;
119 132 : this.groupIndexer = groupIndexer;
120 132 : this.projectCache = projectCache;
121 132 : }
122 :
123 : public ProjectResetter build(ProjectResetter.Config input) throws IOException {
124 132 : return new ProjectResetter(
125 : repoManager,
126 : allUsersName,
127 : accountCreator,
128 : accountCache,
129 : accountIndexer,
130 : groupCache,
131 : groupIncludeCache,
132 : groupIndexer,
133 : projectCache,
134 : input.refsByProject);
135 : }
136 : }
137 :
138 : public static class Config {
139 : private final Multimap<Project.NameKey, String> refsByProject;
140 :
141 132 : public Config() {
142 132 : this.refsByProject = MultimapBuilder.hashKeys().arrayListValues().build();
143 132 : }
144 :
145 : public Config reset(Project.NameKey project, String... refPatterns) {
146 131 : List<String> refPatternList = Arrays.asList(refPatterns);
147 131 : if (refPatternList.isEmpty()) {
148 1 : refPatternList = ImmutableList.of(RefNames.REFS + "*");
149 : }
150 131 : refsByProject.putAll(project, refPatternList);
151 131 : return this;
152 : }
153 : }
154 :
155 : @Inject private GitRepositoryManager repoManager;
156 : @Inject private AllUsersName allUsersName;
157 : @Inject @Nullable private AccountCreator accountCreator;
158 : @Inject @Nullable private AccountCache accountCache;
159 : @Inject @Nullable private GroupCache groupCache;
160 : @Inject @Nullable private GroupIncludeCache groupIncludeCache;
161 : @Inject @Nullable private GroupIndexer groupIndexer;
162 : @Inject @Nullable private AccountIndexer accountIndexer;
163 : @Inject @Nullable private ProjectCache projectCache;
164 :
165 : private final Multimap<Project.NameKey, String> refsPatternByProject;
166 :
167 : // State to which to reset to.
168 : private final Multimap<Project.NameKey, RefState> savedRefStatesByProject;
169 :
170 : // Results of the resetting
171 : private Multimap<Project.NameKey, String> keptRefsByProject;
172 : private Multimap<Project.NameKey, String> restoredRefsByProject;
173 : private Multimap<Project.NameKey, String> deletedRefsByProject;
174 :
175 : private ProjectResetter(
176 : GitRepositoryManager repoManager,
177 : AllUsersName allUsersName,
178 : @Nullable AccountCreator accountCreator,
179 : @Nullable AccountCache accountCache,
180 : @Nullable AccountIndexer accountIndexer,
181 : @Nullable GroupCache groupCache,
182 : @Nullable GroupIncludeCache groupIncludeCache,
183 : @Nullable GroupIndexer groupIndexer,
184 : @Nullable ProjectCache projectCache,
185 : Multimap<Project.NameKey, String> refPatternByProject)
186 132 : throws IOException {
187 132 : this.repoManager = repoManager;
188 132 : this.allUsersName = allUsersName;
189 132 : this.accountCreator = accountCreator;
190 132 : this.accountCache = accountCache;
191 132 : this.accountIndexer = accountIndexer;
192 132 : this.groupCache = groupCache;
193 132 : this.groupIndexer = groupIndexer;
194 132 : this.groupIncludeCache = groupIncludeCache;
195 132 : this.projectCache = projectCache;
196 132 : this.refsPatternByProject = refPatternByProject;
197 132 : this.savedRefStatesByProject = readRefStates();
198 132 : }
199 :
200 : @Override
201 : public void close() throws Exception {
202 132 : keptRefsByProject = MultimapBuilder.hashKeys().arrayListValues().build();
203 132 : restoredRefsByProject = MultimapBuilder.hashKeys().arrayListValues().build();
204 132 : deletedRefsByProject = MultimapBuilder.hashKeys().arrayListValues().build();
205 :
206 132 : restoreRefs();
207 132 : deleteNewlyCreatedRefs();
208 132 : evictCachesAndReindex();
209 132 : }
210 :
211 : /** Read the states of all matching refs. */
212 : private Multimap<Project.NameKey, RefState> readRefStates() throws IOException {
213 : Multimap<Project.NameKey, RefState> refStatesByProject =
214 132 : MultimapBuilder.hashKeys().arrayListValues().build();
215 : for (Map.Entry<Project.NameKey, Collection<String>> e :
216 132 : refsPatternByProject.asMap().entrySet()) {
217 131 : try (Repository repo = repoManager.openRepository(e.getKey())) {
218 131 : Collection<Ref> refs = repo.getRefDatabase().getRefs();
219 131 : for (String refPattern : e.getValue()) {
220 131 : RefPatternMatcher matcher = RefPatternMatcher.getMatcher(refPattern);
221 131 : for (Ref ref : refs) {
222 131 : if (matcher.match(ref.getName(), null)) {
223 131 : refStatesByProject.put(e.getKey(), RefState.create(ref.getName(), ref.getObjectId()));
224 : }
225 131 : }
226 131 : }
227 : }
228 131 : }
229 132 : return refStatesByProject;
230 : }
231 :
232 : private void restoreRefs() throws IOException {
233 : for (Map.Entry<Project.NameKey, Collection<RefState>> e :
234 132 : savedRefStatesByProject.asMap().entrySet()) {
235 131 : try (Repository repo = repoManager.openRepository(e.getKey())) {
236 131 : for (RefState refState : e.getValue()) {
237 131 : if (refState.match(repo)) {
238 131 : keptRefsByProject.put(e.getKey(), refState.ref());
239 131 : continue;
240 : }
241 57 : Ref ref = repo.exactRef(refState.ref());
242 57 : RefUpdate updateRef = repo.updateRef(refState.ref());
243 57 : updateRef.setExpectedOldObjectId(ref != null ? ref.getObjectId() : ObjectId.zeroId());
244 57 : updateRef.setNewObjectId(refState.id());
245 57 : updateRef.setForceUpdate(true);
246 57 : RefUpdate.Result result = updateRef.update();
247 57 : checkState(
248 : result == RefUpdate.Result.FORCED || result == RefUpdate.Result.NEW,
249 : "resetting branch %s in %s failed",
250 57 : refState.ref(),
251 57 : e.getKey());
252 57 : restoredRefsByProject.put(e.getKey(), refState.ref());
253 57 : }
254 : }
255 131 : }
256 132 : }
257 :
258 : private void deleteNewlyCreatedRefs() throws IOException {
259 : for (Map.Entry<Project.NameKey, Collection<String>> e :
260 132 : refsPatternByProject.asMap().entrySet()) {
261 131 : try (Repository repo = repoManager.openRepository(e.getKey())) {
262 131 : Collection<Ref> nonRestoredRefs =
263 131 : repo.getRefDatabase().getRefs().stream()
264 131 : .filter(
265 : r ->
266 131 : !keptRefsByProject.containsEntry(e.getKey(), r.getName())
267 131 : && !restoredRefsByProject.containsEntry(e.getKey(), r.getName()))
268 131 : .collect(toSet());
269 131 : for (String refPattern : e.getValue()) {
270 131 : RefPatternMatcher matcher = RefPatternMatcher.getMatcher(refPattern);
271 131 : for (Ref ref : nonRestoredRefs) {
272 131 : if (matcher.match(ref.getName(), null)
273 44 : && !deletedRefsByProject.containsEntry(e.getKey(), ref.getName())) {
274 44 : RefUpdate updateRef = repo.updateRef(ref.getName());
275 44 : updateRef.setExpectedOldObjectId(ref.getObjectId());
276 44 : updateRef.setNewObjectId(ObjectId.zeroId());
277 44 : updateRef.setForceUpdate(true);
278 44 : RefUpdate.Result result = updateRef.delete();
279 44 : checkState(
280 : result == RefUpdate.Result.FORCED,
281 : "deleting branch %s in %s failed",
282 44 : ref.getName(),
283 44 : e.getKey());
284 44 : deletedRefsByProject.put(e.getKey(), ref.getName());
285 : }
286 131 : }
287 131 : }
288 : }
289 131 : }
290 132 : }
291 :
292 : private void evictCachesAndReindex() throws IOException {
293 132 : evictAndReindexProjects();
294 132 : evictAndReindexAccounts();
295 132 : evictAndReindexGroups();
296 :
297 : // TODO(ekempin): Reindex changes if starred-changes refs in All-Users were modified.
298 132 : }
299 :
300 : /** Evict projects for which the config was changed. */
301 : private void evictAndReindexProjects() {
302 132 : if (projectCache == null) {
303 1 : return;
304 : }
305 :
306 : for (Project.NameKey project :
307 132 : Sets.union(
308 132 : projectsWithConfigChanges(restoredRefsByProject),
309 132 : projectsWithConfigChanges(deletedRefsByProject))) {
310 39 : projectCache.evictAndReindex(project);
311 39 : }
312 132 : }
313 :
314 : private Set<Project.NameKey> projectsWithConfigChanges(
315 : Multimap<Project.NameKey, String> projects) {
316 132 : return projects.entries().stream()
317 132 : .filter(e -> e.getValue().equals(RefNames.REFS_CONFIG))
318 132 : .map(Map.Entry::getKey)
319 132 : .collect(toSet());
320 : }
321 :
322 : /** Evict accounts that were modified. */
323 : private void evictAndReindexAccounts() throws IOException {
324 132 : Set<Account.Id> deletedAccounts = accountIds(deletedRefsByProject.get(allUsersName).stream());
325 132 : if (accountCreator != null) {
326 132 : accountCreator.evict(deletedAccounts);
327 : }
328 132 : if (accountCache != null || accountIndexer != null) {
329 132 : Set<Account.Id> modifiedAccounts =
330 132 : new HashSet<>(accountIds(restoredRefsByProject.get(allUsersName).stream()));
331 :
332 132 : if (restoredRefsByProject.get(allUsersName).contains(RefNames.REFS_EXTERNAL_IDS)
333 131 : || deletedRefsByProject.get(allUsersName).contains(RefNames.REFS_EXTERNAL_IDS)) {
334 : // The external IDs have been modified but we don't know which accounts were affected.
335 : // Make sure all accounts are evicted and reindexed.
336 38 : try (Repository repo = repoManager.openRepository(allUsersName)) {
337 38 : for (Account.Id id : accountIds(repo)) {
338 38 : reindexAccount(id);
339 38 : }
340 : }
341 :
342 : // Remove deleted accounts from the cache and index.
343 38 : for (Account.Id id : deletedAccounts) {
344 36 : reindexAccount(id);
345 38 : }
346 : } else {
347 : // Evict and reindex all modified and deleted accounts.
348 131 : for (Account.Id id : Sets.union(modifiedAccounts, deletedAccounts)) {
349 21 : reindexAccount(id);
350 21 : }
351 : }
352 : }
353 132 : }
354 :
355 : /** Evict groups that were modified. */
356 : private void evictAndReindexGroups() {
357 132 : if (groupCache != null || groupIndexer != null) {
358 132 : Set<AccountGroup.UUID> modifiedGroups =
359 132 : new HashSet<>(groupUUIDs(restoredRefsByProject.get(allUsersName)));
360 132 : Set<AccountGroup.UUID> deletedGroups =
361 132 : new HashSet<>(groupUUIDs(deletedRefsByProject.get(allUsersName)));
362 :
363 : // Evict and reindex all modified and deleted groups.
364 132 : for (AccountGroup.UUID uuid : Sets.union(modifiedGroups, deletedGroups)) {
365 31 : evictAndReindexGroup(uuid);
366 31 : }
367 : }
368 132 : }
369 :
370 : private void reindexAccount(Account.Id accountId) {
371 42 : if (groupIncludeCache != null) {
372 42 : groupIncludeCache.evictGroupsWithMember(accountId);
373 : }
374 42 : if (accountIndexer != null) {
375 42 : accountIndexer.index(accountId);
376 : }
377 42 : }
378 :
379 : private void evictAndReindexGroup(AccountGroup.UUID uuid) {
380 31 : if (groupCache != null) {
381 31 : groupCache.evict(uuid);
382 : }
383 :
384 31 : if (groupIncludeCache != null) {
385 31 : groupIncludeCache.evictParentGroupsOf(uuid);
386 : }
387 :
388 31 : if (groupIndexer != null) {
389 31 : groupIndexer.index(uuid);
390 : }
391 31 : }
392 :
393 : private static Set<Account.Id> accountIds(Repository repo) throws IOException {
394 38 : return accountIds(repo.getRefDatabase().getRefsByPrefix(REFS_USERS).stream().map(Ref::getName));
395 : }
396 :
397 : private static Set<Account.Id> accountIds(Stream<String> refs) {
398 132 : return refs.filter(r -> r.startsWith(REFS_USERS))
399 132 : .map(Account.Id::fromRef)
400 132 : .filter(Objects::nonNull)
401 132 : .collect(toSet());
402 : }
403 :
404 : private Set<AccountGroup.UUID> groupUUIDs(Collection<String> refs) {
405 132 : return refs.stream()
406 132 : .filter(RefNames::isRefsGroups)
407 132 : .map(AccountGroup.UUID::fromRef)
408 132 : .filter(Objects::nonNull)
409 132 : .collect(toSet());
410 : }
411 : }
|
