Line data Source code
1 : // Copyright (C) 2017 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.account.externalids;
16 :
17 : import static com.google.common.base.Preconditions.checkState;
18 : import static java.nio.charset.StandardCharsets.UTF_8;
19 : import static java.util.Objects.requireNonNull;
20 : import static java.util.stream.Collectors.toSet;
21 : import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
22 :
23 : import com.google.common.base.Strings;
24 : import com.google.common.collect.ImmutableSet;
25 : import com.google.common.collect.Iterables;
26 : import com.google.common.collect.Sets;
27 : import com.google.common.collect.Streams;
28 : import com.google.common.flogger.FluentLogger;
29 : import com.google.gerrit.common.Nullable;
30 : import com.google.gerrit.entities.Account;
31 : import com.google.gerrit.entities.RefNames;
32 : import com.google.gerrit.extensions.registration.DynamicMap;
33 : import com.google.gerrit.git.ObjectIds;
34 : import com.google.gerrit.metrics.Counter0;
35 : import com.google.gerrit.metrics.Description;
36 : import com.google.gerrit.metrics.DisabledMetricMaker;
37 : import com.google.gerrit.metrics.MetricMaker;
38 : import com.google.gerrit.server.account.AccountsUpdate;
39 : import com.google.gerrit.server.config.AllUsersName;
40 : import com.google.gerrit.server.config.AuthConfig;
41 : import com.google.gerrit.server.git.meta.MetaDataUpdate;
42 : import com.google.gerrit.server.git.meta.VersionedMetaData;
43 : import com.google.gerrit.server.index.account.AccountIndexer;
44 : import com.google.gerrit.server.logging.CallerFinder;
45 : import com.google.gerrit.server.update.RetryHelper;
46 : import com.google.inject.Inject;
47 : import com.google.inject.Provider;
48 : import com.google.inject.Singleton;
49 : import java.io.IOException;
50 : import java.util.ArrayList;
51 : import java.util.Collection;
52 : import java.util.Collections;
53 : import java.util.HashSet;
54 : import java.util.List;
55 : import java.util.Optional;
56 : import java.util.Set;
57 : import java.util.function.Function;
58 : import org.eclipse.jgit.errors.ConfigInvalidException;
59 : import org.eclipse.jgit.lib.BlobBasedConfig;
60 : import org.eclipse.jgit.lib.CommitBuilder;
61 : import org.eclipse.jgit.lib.Config;
62 : import org.eclipse.jgit.lib.ObjectId;
63 : import org.eclipse.jgit.lib.ObjectInserter;
64 : import org.eclipse.jgit.lib.Repository;
65 : import org.eclipse.jgit.notes.Note;
66 : import org.eclipse.jgit.notes.NoteMap;
67 : import org.eclipse.jgit.revwalk.RevCommit;
68 : import org.eclipse.jgit.revwalk.RevTree;
69 : import org.eclipse.jgit.revwalk.RevWalk;
70 :
71 : /**
72 : * {@link VersionedMetaData} subclass to update external IDs.
73 : *
74 : * <p>This is a low-level API. Read/write of external IDs should be done through {@link
75 : * com.google.gerrit.server.account.AccountsUpdate} or {@link
76 : * com.google.gerrit.server.account.AccountConfig}.
77 : *
78 : * <p>On load the note map from {@code refs/meta/external-ids} is read, but the external IDs are not
79 : * parsed yet (see {@link #onLoad()}).
80 : *
81 : * <p>After loading the note map callers can access single or all external IDs. Only now the
82 : * requested external IDs are parsed.
83 : *
84 : * <p>After loading the note map callers can stage various external ID updates (insert, upsert,
85 : * delete, replace).
86 : *
87 : * <p>On save the staged external ID updates are performed (see {@link #onSave(CommitBuilder)}).
88 : *
89 : * <p>After committing the external IDs a cache update can be requested which also reindexes the
90 : * accounts for which external IDs have been updated (see {@link
91 : * ExternalIdNotesLoader#updateExternalIdCacheAndMaybeReindexAccounts(ExternalIdNotes,
92 : * Collection)}).
93 : */
94 : public class ExternalIdNotes extends VersionedMetaData {
95 151 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
96 :
97 : private static final int MAX_NOTE_SZ = 1 << 19;
98 :
99 : public abstract static class ExternalIdNotesLoader {
100 : protected final ExternalIdCache externalIdCache;
101 : protected final MetricMaker metricMaker;
102 : protected final AllUsersName allUsersName;
103 : protected final DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors;
104 : protected final ExternalIdFactory externalIdFactory;
105 : protected final AuthConfig authConfig;
106 :
107 : protected ExternalIdNotesLoader(
108 : ExternalIdCache externalIdCache,
109 : MetricMaker metricMaker,
110 : AllUsersName allUsersName,
111 : DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors,
112 : ExternalIdFactory externalIdFactory,
113 151 : AuthConfig authConfig) {
114 151 : this.externalIdCache = externalIdCache;
115 151 : this.metricMaker = metricMaker;
116 151 : this.allUsersName = allUsersName;
117 151 : this.upsertPreprocessors = upsertPreprocessors;
118 151 : this.externalIdFactory = externalIdFactory;
119 151 : this.authConfig = authConfig;
120 151 : }
121 :
122 : /**
123 : * Loads the external ID notes from the current tip of the {@code refs/meta/external-ids}
124 : * branch.
125 : *
126 : * @param allUsersRepo the All-Users repository
127 : */
128 : public abstract ExternalIdNotes load(Repository allUsersRepo)
129 : throws IOException, ConfigInvalidException;
130 :
131 : /**
132 : * Loads the external ID notes from the specified revision of the {@code refs/meta/external-ids}
133 : * branch.
134 : *
135 : * @param allUsersRepo the All-Users repository
136 : * @param rev the revision from which the external ID notes should be loaded, if {@code null}
137 : * the external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
138 : * assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
139 : * external IDs will be empty
140 : */
141 : public abstract ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
142 : throws IOException, ConfigInvalidException;
143 :
144 : /**
145 : * Updates the external ID cache. Subclasses of type {@link Factory} will also reindex the
146 : * accounts for which external IDs were modified, while subclasses of type {@link
147 : * FactoryNoReindex} will skip this.
148 : *
149 : * <p>Must only be called after committing changes.
150 : *
151 : * @param externalIdNotes the committed updates that should be applied to the cache. This first
152 : * and last element must be the updates commited first and last, respectively.
153 : * @param accountsToSkipForReindex accounts that should not be reindexed. This is to avoid
154 : * double reindexing when updated accounts will already be reindexed by
155 : * ReindexAfterRefUpdate.
156 : */
157 : public void updateExternalIdCacheAndMaybeReindexAccounts(
158 : ExternalIdNotes externalIdNotes, Collection<Account.Id> accountsToSkipForReindex)
159 : throws IOException {
160 151 : checkState(externalIdNotes.oldRev != null, "no changes committed yet");
161 :
162 : // readOnly is ignored here (legacy behavior).
163 :
164 : // Aggregate all updates.
165 151 : ExternalIdCacheUpdates updates = new ExternalIdCacheUpdates();
166 151 : for (CacheUpdate cacheUpdate : externalIdNotes.cacheUpdates) {
167 151 : cacheUpdate.execute(updates);
168 151 : }
169 :
170 : // Reindex accounts (if the subclass implements reindexAccount()).
171 151 : if (!externalIdNotes.noReindex) {
172 151 : Streams.concat(updates.getAdded().stream(), updates.getRemoved().stream())
173 151 : .map(ExternalId::accountId)
174 151 : .filter(i -> !accountsToSkipForReindex.contains(i))
175 151 : .distinct()
176 151 : .forEach(this::reindexAccount);
177 : }
178 :
179 : // Reset instance state.
180 151 : externalIdNotes.cacheUpdates.clear();
181 151 : externalIdNotes.keysToAdd.clear();
182 151 : externalIdNotes.oldRev = null;
183 151 : }
184 :
185 : protected abstract void reindexAccount(Account.Id id);
186 : }
187 :
188 : @Singleton
189 : public static class Factory extends ExternalIdNotesLoader {
190 :
191 : private final Provider<AccountIndexer> accountIndexer;
192 :
193 : @Inject
194 : Factory(
195 : ExternalIdCache externalIdCache,
196 : Provider<AccountIndexer> accountIndexer,
197 : MetricMaker metricMaker,
198 : AllUsersName allUsersName,
199 : DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors,
200 : ExternalIdFactory externalIdFactory,
201 : AuthConfig authConfig) {
202 151 : super(
203 : externalIdCache,
204 : metricMaker,
205 : allUsersName,
206 : upsertPreprocessors,
207 : externalIdFactory,
208 : authConfig);
209 151 : this.accountIndexer = accountIndexer;
210 151 : }
211 :
212 : @Override
213 : public ExternalIdNotes load(Repository allUsersRepo)
214 : throws IOException, ConfigInvalidException {
215 4 : return new ExternalIdNotes(
216 : metricMaker,
217 : allUsersName,
218 : allUsersRepo,
219 : upsertPreprocessors,
220 : externalIdFactory,
221 4 : authConfig.isUserNameCaseInsensitiveMigrationMode())
222 4 : .load();
223 : }
224 :
225 : @Override
226 : public ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
227 : throws IOException, ConfigInvalidException {
228 151 : return new ExternalIdNotes(
229 : metricMaker,
230 : allUsersName,
231 : allUsersRepo,
232 : upsertPreprocessors,
233 : externalIdFactory,
234 151 : authConfig.isUserNameCaseInsensitiveMigrationMode())
235 151 : .load(rev);
236 : }
237 :
238 : @Override
239 : protected void reindexAccount(Account.Id id) {
240 1 : accountIndexer.get().index(id);
241 1 : }
242 : }
243 :
244 : @Singleton
245 : public static class FactoryNoReindex extends ExternalIdNotesLoader {
246 :
247 : @Inject
248 : FactoryNoReindex(
249 : ExternalIdCache externalIdCache,
250 : MetricMaker metricMaker,
251 : AllUsersName allUsersName,
252 : DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors,
253 : ExternalIdFactory externalIdFactory,
254 : AuthConfig authConfig) {
255 138 : super(
256 : externalIdCache,
257 : metricMaker,
258 : allUsersName,
259 : upsertPreprocessors,
260 : externalIdFactory,
261 : authConfig);
262 138 : }
263 :
264 : @Override
265 : public ExternalIdNotes load(Repository allUsersRepo)
266 : throws IOException, ConfigInvalidException {
267 2 : return new ExternalIdNotes(
268 : metricMaker,
269 : allUsersName,
270 : allUsersRepo,
271 : upsertPreprocessors,
272 : externalIdFactory,
273 2 : authConfig.isUserNameCaseInsensitiveMigrationMode())
274 2 : .setNoReindex()
275 2 : .load();
276 : }
277 :
278 : @Override
279 : public ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
280 : throws IOException, ConfigInvalidException {
281 0 : return new ExternalIdNotes(
282 : metricMaker,
283 : allUsersName,
284 : allUsersRepo,
285 : upsertPreprocessors,
286 : externalIdFactory,
287 0 : authConfig.isUserNameCaseInsensitiveMigrationMode())
288 0 : .setNoReindex()
289 0 : .load(rev);
290 : }
291 :
292 : @Override
293 : protected void reindexAccount(Account.Id id) {
294 : // Do not reindex.
295 0 : }
296 : }
297 :
298 : /**
299 : * Loads the external ID notes for reading only. The external ID notes are loaded from the
300 : * specified revision of the {@code refs/meta/external-ids} branch.
301 : *
302 : * @param rev the revision from which the external ID notes should be loaded, if {@code null} the
303 : * external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
304 : * assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
305 : * external IDs will be empty
306 : * @return read-only {@link ExternalIdNotes} instance
307 : */
308 : public static ExternalIdNotes loadReadOnly(
309 : AllUsersName allUsersName,
310 : Repository allUsersRepo,
311 : @Nullable ObjectId rev,
312 : ExternalIdFactory externalIdFactory,
313 : boolean isUserNameCaseInsensitiveMigrationMode)
314 : throws IOException, ConfigInvalidException {
315 151 : return new ExternalIdNotes(
316 : new DisabledMetricMaker(),
317 : allUsersName,
318 : allUsersRepo,
319 151 : DynamicMap.emptyMap(),
320 : externalIdFactory,
321 : isUserNameCaseInsensitiveMigrationMode)
322 151 : .setReadOnly()
323 151 : .setNoReindex()
324 151 : .load(rev);
325 : }
326 :
327 : /**
328 : * Loads the external ID notes for updates. The external ID notes are loaded from the current tip
329 : * of the {@code refs/meta/external-ids} branch.
330 : *
331 : * <p>Use this only from init, schema upgrades and tests.
332 : *
333 : * <p>Metrics are disabled.
334 : *
335 : * @return {@link ExternalIdNotes} instance that doesn't updates caches on save
336 : */
337 : public static ExternalIdNotes load(
338 : AllUsersName allUsersName,
339 : Repository allUsersRepo,
340 : ExternalIdFactory externalIdFactory,
341 : boolean isUserNameCaseInsensitiveMigrationMode)
342 : throws IOException, ConfigInvalidException {
343 3 : return new ExternalIdNotes(
344 : new DisabledMetricMaker(),
345 : allUsersName,
346 : allUsersRepo,
347 3 : DynamicMap.emptyMap(),
348 : externalIdFactory,
349 : isUserNameCaseInsensitiveMigrationMode)
350 3 : .setNoReindex()
351 3 : .load();
352 : }
353 :
354 : private final AllUsersName allUsersName;
355 : private final Counter0 updateCount;
356 : private final Repository repo;
357 : private final DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors;
358 : private final CallerFinder callerFinder;
359 : private final ExternalIdFactory externalIdFactory;
360 :
361 : private NoteMap noteMap;
362 : private ObjectId oldRev;
363 :
364 : /** Staged note map updates that should be executed on save. */
365 151 : private final List<NoteMapUpdate> noteMapUpdates = new ArrayList<>();
366 :
367 : /** Staged cache updates that should be executed after external ID changes have been committed. */
368 151 : private final List<CacheUpdate> cacheUpdates = new ArrayList<>();
369 :
370 : /**
371 : * When performing batch updates (cf. {@link AccountsUpdate#updateBatch(List)} we need to ensure
372 : * the batch does not introduce duplicates. In addition to checking against the status quo in
373 : * {@link #noteMap} (cf. {@link #checkExternalIdKeysDontExist(Collection)}), which is sufficient
374 : * for single updates, we also need to check for duplicates among the batch updates. As the actual
375 : * updates are computed lazily just before applying them, we unfortunately need to track keys
376 : * explicitly here even though they are already implicit in the lambdas that constitute the
377 : * updates.
378 : */
379 151 : private final Set<ExternalId.Key> keysToAdd = new HashSet<>();
380 :
381 : private Runnable afterReadRevision;
382 151 : private boolean readOnly = false;
383 151 : private boolean noReindex = false;
384 151 : private boolean isUserNameCaseInsensitiveMigrationMode = false;
385 151 : protected final Function<ExternalId, ObjectId> defaultNoteIdResolver =
386 : (extId) -> {
387 151 : ObjectId noteId = extId.key().sha1();
388 : try {
389 151 : if (isUserNameCaseInsensitiveMigrationMode && !noteMap.contains(noteId)) {
390 2 : noteId = extId.key().caseSensitiveSha1();
391 : }
392 0 : } catch (IOException e) {
393 0 : return noteId;
394 151 : }
395 151 : return noteId;
396 : };
397 :
398 : private ExternalIdNotes(
399 : MetricMaker metricMaker,
400 : AllUsersName allUsersName,
401 : Repository allUsersRepo,
402 : DynamicMap<ExternalIdUpsertPreprocessor> upsertPreprocessors,
403 : ExternalIdFactory externalIdFactory,
404 151 : boolean isUserNameCaseInsensitiveMigrationMode) {
405 151 : this.updateCount =
406 151 : metricMaker.newCounter(
407 : "notedb/external_id_update_count",
408 151 : new Description("Total number of external ID updates.").setRate().setUnit("updates"));
409 151 : this.allUsersName = requireNonNull(allUsersName, "allUsersRepo");
410 151 : this.repo = requireNonNull(allUsersRepo, "allUsersRepo");
411 151 : this.upsertPreprocessors = upsertPreprocessors;
412 151 : this.callerFinder =
413 151 : CallerFinder.builder()
414 : // 1. callers that come through ExternalIds
415 151 : .addTarget(ExternalIds.class)
416 :
417 : // 2. callers that come through AccountsUpdate
418 151 : .addTarget(AccountsUpdate.class)
419 151 : .addIgnoredPackage("com.github.rholder.retry")
420 151 : .addIgnoredClass(RetryHelper.class)
421 :
422 : // 3. direct callers
423 151 : .addTarget(ExternalIdNotes.class)
424 151 : .build();
425 151 : this.externalIdFactory = externalIdFactory;
426 151 : this.isUserNameCaseInsensitiveMigrationMode = isUserNameCaseInsensitiveMigrationMode;
427 151 : }
428 :
429 : public ExternalIdNotes setAfterReadRevision(Runnable afterReadRevision) {
430 0 : this.afterReadRevision = afterReadRevision;
431 0 : return this;
432 : }
433 :
434 : private ExternalIdNotes setReadOnly() {
435 151 : readOnly = true;
436 151 : return this;
437 : }
438 :
439 : private ExternalIdNotes setNoReindex() {
440 151 : noReindex = true;
441 151 : return this;
442 : }
443 :
444 : public Repository getRepository() {
445 1 : return repo;
446 : }
447 :
448 : @Override
449 : protected String getRefName() {
450 151 : return RefNames.REFS_EXTERNAL_IDS;
451 : }
452 :
453 : /**
454 : * Loads the external ID notes from the current tip of the {@code refs/meta/external-ids} branch.
455 : *
456 : * @return {@link ExternalIdNotes} instance for chaining
457 : */
458 : private ExternalIdNotes load() throws IOException, ConfigInvalidException {
459 6 : load(allUsersName, repo);
460 6 : return this;
461 : }
462 :
463 : /**
464 : * Loads the external ID notes from the specified revision of the {@code refs/meta/external-ids}
465 : * branch.
466 : *
467 : * @param rev the revision from which the external ID notes should be loaded, if {@code null} the
468 : * external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
469 : * assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
470 : * external IDs will be empty
471 : * @return {@link ExternalIdNotes} instance for chaining
472 : */
473 : ExternalIdNotes load(@Nullable ObjectId rev) throws IOException, ConfigInvalidException {
474 151 : if (rev == null) {
475 3 : return load();
476 : }
477 151 : if (ObjectId.zeroId().equals(rev)) {
478 151 : load(allUsersName, repo, null);
479 151 : return this;
480 : }
481 151 : load(allUsersName, repo, rev);
482 151 : return this;
483 : }
484 :
485 : /**
486 : * Parses and returns the specified external ID.
487 : *
488 : * @param key the key of the external ID
489 : * @return the external ID, {@code Optional.empty()} if it doesn't exist
490 : */
491 : public Optional<ExternalId> get(ExternalId.Key key) throws IOException, ConfigInvalidException {
492 10 : checkLoaded();
493 10 : ObjectId noteId = getNoteId(key);
494 10 : if (noteMap.contains(noteId)) {
495 :
496 10 : try (RevWalk rw = new RevWalk(repo)) {
497 10 : ObjectId noteDataId = noteMap.get(noteId);
498 10 : byte[] raw = readNoteData(rw, noteDataId);
499 10 : return Optional.of(externalIdFactory.parse(noteId.name(), raw, noteDataId));
500 : }
501 : }
502 4 : return Optional.empty();
503 : }
504 :
505 : protected ObjectId getNoteId(ExternalId.Key key) throws IOException {
506 17 : ObjectId noteId = key.sha1();
507 :
508 17 : if (!noteMap.contains(noteId) && isUserNameCaseInsensitiveMigrationMode) {
509 2 : noteId = key.caseSensitiveSha1();
510 : }
511 :
512 17 : return noteId;
513 : }
514 :
515 : /**
516 : * Parses and returns the specified external IDs.
517 : *
518 : * @param keys the keys of the external IDs
519 : * @return the external IDs
520 : */
521 : public Set<ExternalId> get(Collection<ExternalId.Key> keys)
522 : throws IOException, ConfigInvalidException {
523 151 : checkLoaded();
524 151 : HashSet<ExternalId> externalIds = Sets.newHashSetWithExpectedSize(keys.size());
525 151 : for (ExternalId.Key key : keys) {
526 9 : get(key).ifPresent(externalIds::add);
527 9 : }
528 151 : return externalIds;
529 : }
530 :
531 : /**
532 : * Parses and returns all external IDs.
533 : *
534 : * <p>Invalid external IDs are ignored.
535 : *
536 : * @return all external IDs
537 : */
538 : public ImmutableSet<ExternalId> all() throws IOException {
539 151 : checkLoaded();
540 151 : try (RevWalk rw = new RevWalk(repo)) {
541 151 : ImmutableSet.Builder<ExternalId> b = ImmutableSet.builder();
542 151 : for (Note note : noteMap) {
543 151 : byte[] raw = readNoteData(rw, note.getData());
544 : try {
545 151 : b.add(externalIdFactory.parse(note.getName(), raw, note.getData()));
546 2 : } catch (ConfigInvalidException | RuntimeException e) {
547 2 : logger.atSevere().withCause(e).log(
548 2 : "Ignoring invalid external ID note %s", note.getName());
549 151 : }
550 151 : }
551 151 : return b.build();
552 : }
553 : }
554 :
555 : NoteMap getNoteMap() {
556 1 : checkLoaded();
557 1 : return noteMap;
558 : }
559 :
560 : static byte[] readNoteData(RevWalk rw, ObjectId noteDataId) throws IOException {
561 151 : return rw.getObjectReader().open(noteDataId, OBJ_BLOB).getCachedBytes(MAX_NOTE_SZ);
562 : }
563 :
564 : /**
565 : * Inserts a new external ID.
566 : *
567 : * @throws IOException on IO error while checking if external ID already exists
568 : * @throws DuplicateExternalIdKeyException if the external ID already exists
569 : */
570 : public void insert(ExternalId extId) throws IOException, DuplicateExternalIdKeyException {
571 6 : insert(Collections.singleton(extId));
572 6 : }
573 :
574 : /**
575 : * Inserts new external IDs.
576 : *
577 : * @throws IOException on IO error while checking if external IDs already exist
578 : * @throws DuplicateExternalIdKeyException if any of the external ID already exists
579 : */
580 : public void insert(Collection<ExternalId> extIds)
581 : throws IOException, DuplicateExternalIdKeyException {
582 6 : checkLoaded();
583 6 : checkExternalIdsDontExist(extIds);
584 :
585 6 : Set<ExternalId> newExtIds = new HashSet<>();
586 6 : noteMapUpdates.add(
587 : (rw, n) -> {
588 6 : for (ExternalId extId : extIds) {
589 6 : ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId);
590 6 : preprocessUpsert(insertedExtId);
591 6 : newExtIds.add(insertedExtId);
592 6 : }
593 6 : });
594 6 : cacheUpdates.add(cu -> cu.add(newExtIds));
595 6 : incrementalDuplicateDetection(extIds);
596 6 : }
597 :
598 : /**
599 : * Inserts or updates an external ID.
600 : *
601 : * <p>If the external ID already exists, it is overwritten, otherwise it is inserted.
602 : */
603 : public void upsert(ExternalId extId) throws IOException, ConfigInvalidException {
604 4 : upsert(Collections.singleton(extId));
605 4 : }
606 :
607 : /**
608 : * Inserts or updates external IDs.
609 : *
610 : * <p>If any of the external IDs already exists, it is overwritten. New external IDs are inserted.
611 : */
612 : public void upsert(Collection<ExternalId> extIds) throws IOException, ConfigInvalidException {
613 151 : checkLoaded();
614 151 : Set<ExternalId> removedExtIds = get(ExternalId.Key.from(extIds));
615 151 : Set<ExternalId> updatedExtIds = new HashSet<>();
616 151 : noteMapUpdates.add(
617 : (rw, n) -> {
618 151 : for (ExternalId extId : extIds) {
619 9 : ExternalId updatedExtId = upsert(rw, inserter, noteMap, extId);
620 9 : preprocessUpsert(updatedExtId);
621 9 : updatedExtIds.add(updatedExtId);
622 9 : }
623 151 : });
624 151 : cacheUpdates.add(cu -> cu.remove(removedExtIds).add(updatedExtIds));
625 151 : incrementalDuplicateDetection(extIds);
626 151 : }
627 :
628 : /**
629 : * Deletes an external ID.
630 : *
631 : * @throws IllegalStateException is thrown if there is an existing external ID that has the same
632 : * key, but otherwise doesn't match the specified external ID.
633 : */
634 : public void delete(ExternalId extId) {
635 1 : delete(Collections.singleton(extId));
636 1 : }
637 :
638 : /**
639 : * Deletes external IDs.
640 : *
641 : * @throws IllegalStateException is thrown if there is an existing external ID that has the same
642 : * key as any of the external IDs that should be deleted, but otherwise doesn't match the that
643 : * external ID.
644 : */
645 : public void delete(Collection<ExternalId> extIds) {
646 1 : checkLoaded();
647 1 : Set<ExternalId> removedExtIds = new HashSet<>();
648 1 : noteMapUpdates.add(
649 : (rw, n) -> {
650 1 : for (ExternalId extId : extIds) {
651 1 : remove(rw, noteMap, extId);
652 1 : removedExtIds.add(extId);
653 1 : }
654 1 : });
655 1 : cacheUpdates.add(cu -> cu.remove(removedExtIds));
656 1 : }
657 :
658 : /**
659 : * Delete an external ID by key.
660 : *
661 : * @throws IllegalStateException is thrown if the external ID does not belong to the specified
662 : * account.
663 : */
664 : public void delete(Account.Id accountId, ExternalId.Key extIdKey) {
665 1 : delete(accountId, Collections.singleton(extIdKey));
666 1 : }
667 :
668 : /**
669 : * Delete external IDs by external ID key.
670 : *
671 : * @throws IllegalStateException is thrown if any of the external IDs does not belong to the
672 : * specified account.
673 : */
674 : public void delete(Account.Id accountId, Collection<ExternalId.Key> extIdKeys) {
675 1 : checkLoaded();
676 1 : Set<ExternalId> removedExtIds = new HashSet<>();
677 1 : noteMapUpdates.add(
678 : (rw, n) -> {
679 1 : for (ExternalId.Key extIdKey : extIdKeys) {
680 1 : ExternalId removedExtId = remove(rw, noteMap, extIdKey, accountId);
681 1 : removedExtIds.add(removedExtId);
682 1 : }
683 1 : });
684 1 : cacheUpdates.add(cu -> cu.remove(removedExtIds));
685 1 : }
686 :
687 : /**
688 : * Delete external IDs by external ID key.
689 : *
690 : * <p>The external IDs are deleted regardless of which account they belong to.
691 : */
692 : public void deleteByKeys(Collection<ExternalId.Key> extIdKeys) {
693 0 : checkLoaded();
694 0 : Set<ExternalId> removedExtIds = new HashSet<>();
695 0 : noteMapUpdates.add(
696 : (rw, n) -> {
697 0 : for (ExternalId.Key extIdKey : extIdKeys) {
698 0 : ExternalId extId = remove(rw, noteMap, extIdKey, null);
699 0 : removedExtIds.add(extId);
700 0 : }
701 0 : });
702 0 : cacheUpdates.add(cu -> cu.remove(removedExtIds));
703 0 : }
704 :
705 : public void replace(
706 : Account.Id accountId, Collection<ExternalId.Key> toDelete, Collection<ExternalId> toAdd)
707 : throws IOException, DuplicateExternalIdKeyException {
708 151 : replace(accountId, toDelete, toAdd, defaultNoteIdResolver);
709 151 : }
710 :
711 : /**
712 : * Replaces external IDs for an account by external ID keys.
713 : *
714 : * <p>Deletion of external IDs is done before adding the new external IDs. This means if an
715 : * external ID key is specified for deletion and an external ID with the same key is specified to
716 : * be added, the old external ID with that key is deleted first and then the new external ID is
717 : * added (so the external ID for that key is replaced).
718 : *
719 : * @throws IllegalStateException is thrown if any of the specified external IDs does not belong to
720 : * the specified account.
721 : */
722 : public void replace(
723 : Account.Id accountId,
724 : Collection<ExternalId.Key> toDelete,
725 : Collection<ExternalId> toAdd,
726 : Function<ExternalId, ObjectId> noteIdResolver)
727 : throws IOException, DuplicateExternalIdKeyException {
728 151 : checkLoaded();
729 151 : checkSameAccount(toAdd, accountId);
730 151 : checkExternalIdKeysDontExist(ExternalId.Key.from(toAdd), toDelete);
731 :
732 151 : Set<ExternalId> removedExtIds = new HashSet<>();
733 151 : Set<ExternalId> updatedExtIds = new HashSet<>();
734 151 : noteMapUpdates.add(
735 : (rw, n) -> {
736 151 : for (ExternalId.Key extIdKey : toDelete) {
737 13 : ExternalId removedExtId = remove(rw, noteMap, extIdKey, accountId);
738 13 : if (removedExtId != null) {
739 13 : removedExtIds.add(removedExtId);
740 : }
741 13 : }
742 :
743 151 : for (ExternalId extId : toAdd) {
744 151 : ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId, noteIdResolver);
745 151 : preprocessUpsert(insertedExtId);
746 151 : updatedExtIds.add(insertedExtId);
747 151 : }
748 151 : });
749 151 : cacheUpdates.add(cu -> cu.add(updatedExtIds).remove(removedExtIds));
750 151 : incrementalDuplicateDetection(toAdd);
751 151 : }
752 :
753 : /**
754 : * Replaces external IDs for an account by external ID keys.
755 : *
756 : * <p>Deletion of external IDs is done before adding the new external IDs. This means if an
757 : * external ID key is specified for deletion and an external ID with the same key is specified to
758 : * be added, the old external ID with that key is deleted first and then the new external ID is
759 : * added (so the external ID for that key is replaced).
760 : *
761 : * <p>The external IDs are replaced regardless of which account they belong to.
762 : */
763 : public void replaceByKeys(Collection<ExternalId.Key> toDelete, Collection<ExternalId> toAdd)
764 : throws IOException, DuplicateExternalIdKeyException {
765 1 : checkLoaded();
766 1 : checkExternalIdKeysDontExist(ExternalId.Key.from(toAdd), toDelete);
767 :
768 1 : Set<ExternalId> removedExtIds = new HashSet<>();
769 1 : Set<ExternalId> updatedExtIds = new HashSet<>();
770 1 : noteMapUpdates.add(
771 : (rw, n) -> {
772 1 : for (ExternalId.Key extIdKey : toDelete) {
773 1 : ExternalId removedExtId = remove(rw, noteMap, extIdKey, null);
774 1 : removedExtIds.add(removedExtId);
775 1 : }
776 :
777 1 : for (ExternalId extId : toAdd) {
778 1 : ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId);
779 1 : preprocessUpsert(insertedExtId);
780 1 : updatedExtIds.add(insertedExtId);
781 1 : }
782 1 : });
783 1 : cacheUpdates.add(cu -> cu.add(updatedExtIds).remove(removedExtIds));
784 1 : incrementalDuplicateDetection(toAdd);
785 1 : }
786 :
787 : /**
788 : * Replaces an external ID.
789 : *
790 : * @throws IllegalStateException is thrown if the specified external IDs belong to different
791 : * accounts.
792 : */
793 : public void replace(ExternalId toDelete, ExternalId toAdd)
794 : throws IOException, DuplicateExternalIdKeyException {
795 1 : replace(Collections.singleton(toDelete), Collections.singleton(toAdd));
796 1 : }
797 :
798 : /**
799 : * Replaces external IDs.
800 : *
801 : * <p>Deletion of external IDs is done before adding the new external IDs. This means if an
802 : * external ID is specified for deletion and an external ID with the same key is specified to be
803 : * added, the old external ID with that key is deleted first and then the new external ID is added
804 : * (so the external ID for that key is replaced).
805 : *
806 : * @throws IllegalStateException is thrown if the specified external IDs belong to different
807 : * accounts.
808 : */
809 : public void replace(Collection<ExternalId> toDelete, Collection<ExternalId> toAdd)
810 : throws IOException, DuplicateExternalIdKeyException {
811 151 : Account.Id accountId = checkSameAccount(Iterables.concat(toDelete, toAdd));
812 151 : if (accountId == null) {
813 : // toDelete and toAdd are empty -> nothing to do
814 36 : return;
815 : }
816 :
817 151 : replace(accountId, toDelete.stream().map(ExternalId::key).collect(toSet()), toAdd);
818 151 : }
819 :
820 : /**
821 : * Replaces external IDs.
822 : *
823 : * <p>Deletion of external IDs is done before adding the new external IDs. This means if an
824 : * external ID is specified for deletion and an external ID with the same key is specified to be
825 : * added, the old external ID with that key is deleted first and then the new external ID is added
826 : * (so the external ID for that key is replaced).
827 : *
828 : * @throws IllegalStateException is thrown if the specified external IDs belong to different
829 : * accounts.
830 : */
831 : public void replace(
832 : Collection<ExternalId> toDelete,
833 : Collection<ExternalId> toAdd,
834 : Function<ExternalId, ObjectId> noteIdResolver)
835 : throws IOException, DuplicateExternalIdKeyException {
836 2 : Account.Id accountId = checkSameAccount(Iterables.concat(toDelete, toAdd));
837 2 : if (accountId == null) {
838 : // toDelete and toAdd are empty -> nothing to do
839 0 : return;
840 : }
841 :
842 2 : replace(
843 2 : accountId, toDelete.stream().map(ExternalId::key).collect(toSet()), toAdd, noteIdResolver);
844 2 : }
845 :
846 : @Override
847 : protected void onLoad() throws IOException, ConfigInvalidException {
848 151 : if (revision != null) {
849 151 : logger.atFine().log(
850 151 : "Reading external ID note map (caller: %s)", callerFinder.findCallerLazy());
851 151 : noteMap = NoteMap.read(reader, revision);
852 : } else {
853 151 : noteMap = NoteMap.newEmptyMap();
854 : }
855 :
856 151 : if (afterReadRevision != null) {
857 0 : afterReadRevision.run();
858 : }
859 151 : }
860 :
861 : @Override
862 : public RevCommit commit(MetaDataUpdate update) throws IOException {
863 151 : oldRev = ObjectIds.copyOrZero(revision);
864 151 : RevCommit commit = super.commit(update);
865 151 : updateCount.increment();
866 151 : return commit;
867 : }
868 :
869 : @Override
870 : protected boolean onSave(CommitBuilder commit) throws IOException, ConfigInvalidException {
871 151 : checkState(!readOnly, "Updating external IDs is disabled");
872 :
873 151 : if (noteMapUpdates.isEmpty()) {
874 2 : return false;
875 : }
876 :
877 151 : logger.atFine().log("Updating external IDs");
878 :
879 151 : if (Strings.isNullOrEmpty(commit.getMessage())) {
880 6 : commit.setMessage("Update external IDs\n");
881 : }
882 :
883 151 : try (RevWalk rw = new RevWalk(reader)) {
884 151 : for (NoteMapUpdate noteMapUpdate : noteMapUpdates) {
885 : try {
886 151 : noteMapUpdate.execute(rw, noteMap);
887 0 : } catch (DuplicateExternalIdKeyException e) {
888 0 : throw new IOException(e);
889 151 : }
890 151 : }
891 151 : noteMapUpdates.clear();
892 :
893 151 : RevTree oldTree = revision != null ? rw.parseTree(revision) : null;
894 151 : ObjectId newTreeId = noteMap.writeTree(inserter);
895 151 : if (newTreeId.equals(oldTree)) {
896 33 : return false;
897 : }
898 :
899 151 : commit.setTreeId(newTreeId);
900 151 : return true;
901 33 : }
902 : }
903 :
904 : /**
905 : * Checks that all specified external IDs belong to the same account.
906 : *
907 : * @return the ID of the account to which all specified external IDs belong.
908 : */
909 : private static Account.Id checkSameAccount(Iterable<ExternalId> extIds) {
910 151 : return checkSameAccount(extIds, null);
911 : }
912 :
913 : /**
914 : * Checks that all specified external IDs belong to specified account. If no account is specified
915 : * it is checked that all specified external IDs belong to the same account.
916 : *
917 : * @return the ID of the account to which all specified external IDs belong.
918 : */
919 : public static Account.Id checkSameAccount(
920 : Iterable<ExternalId> extIds, @Nullable Account.Id accountId) {
921 151 : for (ExternalId extId : extIds) {
922 151 : if (accountId == null) {
923 151 : accountId = extId.accountId();
924 151 : continue;
925 : }
926 151 : checkState(
927 151 : accountId.equals(extId.accountId()),
928 : "external id %s belongs to account %s, but expected account %s",
929 151 : extId.key().get(),
930 151 : extId.accountId().get(),
931 151 : accountId.get());
932 151 : }
933 151 : return accountId;
934 : }
935 :
936 : private void incrementalDuplicateDetection(Collection<ExternalId> externalIds) {
937 151 : externalIds.stream()
938 151 : .map(ExternalId::key)
939 151 : .forEach(
940 : key -> {
941 151 : if (!keysToAdd.add(key)) {
942 1 : throw new DuplicateExternalIdKeyException(key);
943 : }
944 151 : });
945 151 : }
946 :
947 : /**
948 : * Inserts or updates a new external ID and sets it in the note map.
949 : *
950 : * <p>If the external ID already exists, it is overwritten.
951 : */
952 : private ExternalId upsert(RevWalk rw, ObjectInserter ins, NoteMap noteMap, ExternalId extId)
953 : throws IOException, ConfigInvalidException {
954 11 : return upsert(rw, ins, noteMap, extId, defaultNoteIdResolver);
955 : }
956 :
957 : /**
958 : * Inserts or updates a new external ID and sets it in the note map.
959 : *
960 : * <p>If the external ID already exists, it is overwritten.
961 : */
962 : private ExternalId upsert(
963 : RevWalk rw,
964 : ObjectInserter ins,
965 : NoteMap noteMap,
966 : ExternalId extId,
967 : Function<ExternalId, ObjectId> noteIdResolver)
968 : throws IOException, ConfigInvalidException {
969 151 : ObjectId noteId = extId.key().sha1();
970 151 : Config c = new Config();
971 151 : ObjectId resolvedNoteId = noteIdResolver.apply(extId);
972 151 : if (noteMap.contains(resolvedNoteId)) {
973 9 : noteId = resolvedNoteId;
974 9 : ObjectId noteDataId = noteMap.get(noteId);
975 9 : byte[] raw = readNoteData(rw, noteDataId);
976 : try {
977 9 : c = new BlobBasedConfig(null, raw);
978 0 : } catch (ConfigInvalidException e) {
979 0 : throw new ConfigInvalidException(
980 0 : String.format("Invalid external id config for note %s: %s", noteId, e.getMessage()));
981 9 : }
982 : }
983 151 : extId.writeToConfig(c);
984 151 : byte[] raw = c.toText().getBytes(UTF_8);
985 151 : ObjectId noteData = ins.insert(OBJ_BLOB, raw);
986 151 : noteMap.set(noteId, noteData);
987 151 : return externalIdFactory.create(extId, noteData);
988 : }
989 :
990 : /**
991 : * Removes an external ID from the note map.
992 : *
993 : * @throws IllegalStateException is thrown if there is an existing external ID that has the same
994 : * key, but otherwise doesn't match the specified external ID.
995 : */
996 : private void remove(RevWalk rw, NoteMap noteMap, ExternalId extId)
997 : throws IOException, ConfigInvalidException {
998 1 : ObjectId noteId = getNoteId(extId.key());
999 :
1000 1 : if (!noteMap.contains(noteId)) {
1001 0 : return;
1002 : }
1003 :
1004 1 : ObjectId noteDataId = noteMap.get(noteId);
1005 1 : byte[] raw = readNoteData(rw, noteDataId);
1006 1 : ExternalId actualExtId = externalIdFactory.parse(noteId.name(), raw, noteDataId);
1007 1 : checkState(
1008 1 : extId.equals(actualExtId),
1009 : "external id %s should be removed, but it doesn't match the actual external id %s",
1010 1 : extId.toString(),
1011 1 : actualExtId.toString());
1012 1 : noteMap.remove(noteId);
1013 1 : }
1014 :
1015 : /**
1016 : * Removes an external ID from the note map by external ID key.
1017 : *
1018 : * @throws IllegalStateException is thrown if an expected account ID is provided and an external
1019 : * ID with the specified key exists, but belongs to another account.
1020 : * @return the external ID that was removed, {@code null} if no external ID with the specified key
1021 : * exists
1022 : */
1023 : @Nullable
1024 : private ExternalId remove(
1025 : RevWalk rw, NoteMap noteMap, ExternalId.Key extIdKey, Account.Id expectedAccountId)
1026 : throws IOException, ConfigInvalidException {
1027 13 : ObjectId noteId = getNoteId(extIdKey);
1028 :
1029 13 : if (!noteMap.contains(noteId)) {
1030 0 : return null;
1031 : }
1032 :
1033 13 : ObjectId noteDataId = noteMap.get(noteId);
1034 13 : byte[] raw = readNoteData(rw, noteDataId);
1035 13 : ExternalId extId = externalIdFactory.parse(noteId.name(), raw, noteDataId);
1036 13 : if (expectedAccountId != null) {
1037 13 : checkState(
1038 13 : expectedAccountId.equals(extId.accountId()),
1039 : "external id %s should be removed for account %s,"
1040 : + " but external id belongs to account %s",
1041 13 : extIdKey.get(),
1042 13 : expectedAccountId.get(),
1043 13 : extId.accountId().get());
1044 : }
1045 13 : noteMap.remove(noteId);
1046 13 : return extId;
1047 : }
1048 :
1049 : private void checkExternalIdsDontExist(Collection<ExternalId> extIds)
1050 : throws DuplicateExternalIdKeyException, IOException {
1051 6 : checkExternalIdKeysDontExist(ExternalId.Key.from(extIds));
1052 6 : }
1053 :
1054 : private void checkExternalIdKeysDontExist(
1055 : Collection<ExternalId.Key> extIdKeysToAdd, Collection<ExternalId.Key> extIdKeysToDelete)
1056 : throws DuplicateExternalIdKeyException, IOException {
1057 151 : HashSet<ExternalId.Key> newKeys = new HashSet<>(extIdKeysToAdd);
1058 151 : newKeys.removeAll(extIdKeysToDelete);
1059 151 : checkExternalIdKeysDontExist(newKeys);
1060 151 : }
1061 :
1062 : private void checkExternalIdKeysDontExist(Collection<ExternalId.Key> extIdKeys)
1063 : throws IOException, DuplicateExternalIdKeyException {
1064 151 : for (ExternalId.Key extIdKey : extIdKeys) {
1065 151 : if (noteMap.contains(extIdKey.sha1())) {
1066 4 : throw new DuplicateExternalIdKeyException(extIdKey);
1067 : }
1068 151 : }
1069 151 : }
1070 :
1071 : private void checkLoaded() {
1072 151 : checkState(noteMap != null, "External IDs not loaded yet");
1073 151 : }
1074 :
1075 : private void preprocessUpsert(ExternalId extId) {
1076 151 : upsertPreprocessors.forEach(p -> p.get().upsert(extId));
1077 151 : }
1078 :
1079 : @FunctionalInterface
1080 : private interface NoteMapUpdate {
1081 : void execute(RevWalk rw, NoteMap noteMap)
1082 : throws IOException, ConfigInvalidException, DuplicateExternalIdKeyException;
1083 : }
1084 :
1085 : @FunctionalInterface
1086 : private interface CacheUpdate {
1087 : void execute(ExternalIdCacheUpdates cacheUpdates) throws IOException;
1088 : }
1089 :
1090 151 : private static class ExternalIdCacheUpdates {
1091 151 : final Set<ExternalId> added = new HashSet<>();
1092 151 : final Set<ExternalId> removed = new HashSet<>();
1093 :
1094 : ExternalIdCacheUpdates add(Collection<ExternalId> extIds) {
1095 151 : this.added.addAll(extIds);
1096 151 : return this;
1097 : }
1098 :
1099 : Set<ExternalId> getAdded() {
1100 151 : return ImmutableSet.copyOf(added);
1101 : }
1102 :
1103 : ExternalIdCacheUpdates remove(Collection<ExternalId> extIds) {
1104 151 : this.removed.addAll(extIds);
1105 151 : return this;
1106 : }
1107 :
1108 : Set<ExternalId> getRemoved() {
1109 151 : return ImmutableSet.copyOf(removed);
1110 : }
1111 : }
1112 : }
|
