Line data Source code
1 : // Copyright (C) 2021 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 : package com.google.gerrit.server.notedb;
15 :
16 : import static com.google.common.base.MoreObjects.firstNonNull;
17 : import static com.google.common.base.Preconditions.checkState;
18 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_ASSIGNEE;
19 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_ATTENTION;
20 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_LABEL;
21 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_REAL_USER;
22 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_SUBMITTED_WITH;
23 : import static com.google.gerrit.server.notedb.ChangeNoteFooters.FOOTER_TAG;
24 : import static com.google.gerrit.server.util.AccountTemplateUtil.ACCOUNT_TEMPLATE_PATTERN;
25 : import static com.google.gerrit.server.util.AccountTemplateUtil.ACCOUNT_TEMPLATE_REGEX;
26 : import static java.nio.charset.StandardCharsets.UTF_8;
27 :
28 : import com.google.auto.value.AutoValue;
29 : import com.google.common.base.Splitter;
30 : import com.google.common.base.Strings;
31 : import com.google.common.collect.ImmutableMap;
32 : import com.google.common.collect.ImmutableSet;
33 : import com.google.common.collect.Iterables;
34 : import com.google.common.flogger.FluentLogger;
35 : import com.google.gerrit.common.Nullable;
36 : import com.google.gerrit.common.UsedAt;
37 : import com.google.gerrit.entities.Account;
38 : import com.google.gerrit.entities.Change;
39 : import com.google.gerrit.entities.HumanComment;
40 : import com.google.gerrit.entities.PatchSetApproval;
41 : import com.google.gerrit.entities.Project;
42 : import com.google.gerrit.entities.RefNames;
43 : import com.google.gerrit.entities.SubmitRecord;
44 : import com.google.gerrit.git.RefUpdateUtil;
45 : import com.google.gerrit.json.OutputFormat;
46 : import com.google.gerrit.server.ChangeMessagesUtil;
47 : import com.google.gerrit.server.account.AccountCache;
48 : import com.google.gerrit.server.account.AccountState;
49 : import com.google.gerrit.server.account.externalids.ExternalId;
50 : import com.google.gerrit.server.notedb.ChangeNoteUtil.AttentionStatusInNoteDb;
51 : import com.google.gerrit.server.notedb.ChangeNoteUtil.CommitMessageRange;
52 : import com.google.gerrit.server.util.AccountTemplateUtil;
53 : import com.google.gson.Gson;
54 : import com.google.inject.Inject;
55 : import com.google.inject.Singleton;
56 : import java.io.ByteArrayOutputStream;
57 : import java.io.IOException;
58 : import java.io.Serializable;
59 : import java.nio.charset.Charset;
60 : import java.util.ArrayList;
61 : import java.util.Arrays;
62 : import java.util.Collection;
63 : import java.util.HashMap;
64 : import java.util.HashSet;
65 : import java.util.List;
66 : import java.util.Map;
67 : import java.util.Objects;
68 : import java.util.Optional;
69 : import java.util.Set;
70 : import java.util.regex.Matcher;
71 : import java.util.regex.Pattern;
72 : import java.util.stream.Collectors;
73 : import java.util.stream.Stream;
74 : import org.apache.commons.lang3.StringUtils;
75 : import org.eclipse.jgit.diff.DiffAlgorithm;
76 : import org.eclipse.jgit.diff.DiffFormatter;
77 : import org.eclipse.jgit.diff.EditList;
78 : import org.eclipse.jgit.diff.HistogramDiff;
79 : import org.eclipse.jgit.diff.RawText;
80 : import org.eclipse.jgit.diff.RawTextComparator;
81 : import org.eclipse.jgit.errors.ConfigInvalidException;
82 : import org.eclipse.jgit.internal.storage.file.FileRepository;
83 : import org.eclipse.jgit.internal.storage.file.PackInserter;
84 : import org.eclipse.jgit.lib.BatchRefUpdate;
85 : import org.eclipse.jgit.lib.CommitBuilder;
86 : import org.eclipse.jgit.lib.Constants;
87 : import org.eclipse.jgit.lib.ObjectId;
88 : import org.eclipse.jgit.lib.ObjectInserter;
89 : import org.eclipse.jgit.lib.PersonIdent;
90 : import org.eclipse.jgit.lib.Ref;
91 : import org.eclipse.jgit.lib.Repository;
92 : import org.eclipse.jgit.revwalk.FooterLine;
93 : import org.eclipse.jgit.revwalk.RevCommit;
94 : import org.eclipse.jgit.revwalk.RevSort;
95 : import org.eclipse.jgit.revwalk.RevWalk;
96 : import org.eclipse.jgit.transport.ReceiveCommand;
97 : import org.eclipse.jgit.util.RawParseUtils;
98 :
99 : /**
100 : * Rewrites ('backfills') commit history of change in NoteDb to not contain user data. Only fixes
101 : * known cases, rewriting commits case by case.
102 : *
103 : * <p>The cases where we used to put user data in NoteDb can be found by
104 : * https://gerrit-review.googlesource.com/q/hashtag:user-data-cleanup
105 : *
106 : * <p>As opposed to {@link NoteDbRewriter} implementations, which target a specific change and are
107 : * used by REST endpoints, this rewriter is used as standalone tool, that bulk backfills changes by
108 : * project.
109 : */
110 : @UsedAt(UsedAt.Project.GOOGLE)
111 : @Singleton
112 : public class CommitRewriter {
113 : /** Options to run {@link #backfillProject}. */
114 1 : public static class RunOptions implements Serializable {
115 : private static final long serialVersionUID = 1L;
116 :
117 : /** Whether to rewrite the commit history or only find refs that need to be fixed. */
118 1 : public boolean dryRun = true;
119 : /**
120 : * Whether to verify that resulting commits contain user data for the accounts that are linked
121 : * to a change, see {@link #verifyCommit}, {@link #collectAccounts}.
122 : */
123 1 : public boolean verifyCommits = true;
124 : /** Whether to compute and output the diff of the commit history for the backfilled refs. */
125 1 : public boolean outputDiff = true;
126 :
127 : /** Max number of refs to update in a single {@link BatchRefUpdate}. */
128 1 : public int maxRefsInBatch = 10000;
129 : /**
130 : * Max number of refs to fix by a single {@link RefsUpdate} run. Since the second run on the
131 : * same set of refs is a no-op, running with this option in a loop will eventually fix all refs.
132 : * The number of executed {@link BatchRefUpdate} depends on {@link #maxRefsInBatch} option.
133 : */
134 1 : public int maxRefsToUpdate = 50000;
135 : }
136 :
137 : /** Result of the backfill run for a project. */
138 1 : public static class BackfillResult {
139 :
140 : /** If the run for the project was successful. */
141 : public boolean ok;
142 :
143 : /**
144 : * Refs that were fixed by the run/ would be fixed if in --dry-run, together with their commit
145 : * history diff. Diff is empty if --output-diff is false.
146 : */
147 1 : public Map<String, List<CommitDiff>> fixedRefDiff = new HashMap<>();
148 :
149 : /**
150 : * Refs that still contain user data after the backfill run. Only filled if --verify-commits,
151 : * see {@link #verifyCommit}
152 : */
153 1 : public List<String> refsStillInvalidAfterFix = new ArrayList<>();
154 :
155 : /** Refs, failed to backfill by the run. */
156 1 : public List<String> refsFailedToFix = new ArrayList<>();
157 : }
158 :
159 : /** Diff result of a single commit rewrite */
160 : @AutoValue
161 1 : public abstract static class CommitDiff {
162 : public static CommitDiff create(ObjectId oldSha1, String commitDiff) {
163 1 : return new AutoValue_CommitRewriter_CommitDiff(oldSha1, commitDiff);
164 : }
165 :
166 : /** SHA1 of the overwritten commit */
167 : public abstract ObjectId oldSha1();
168 :
169 : /** Diff applied to the commit with {@link #oldSha1} */
170 : public abstract String diff();
171 : }
172 :
173 : public static final String DEFAULT_ACCOUNT_REPLACEMENT = "Gerrit Account";
174 :
175 1 : private static final Pattern NON_REPLACE_ACCOUNT_PATTERN =
176 1 : Pattern.compile(DEFAULT_ACCOUNT_REPLACEMENT + "|" + ACCOUNT_TEMPLATE_REGEX);
177 :
178 1 : private static final Pattern OK_ACCOUNT_NAME_PATTERN =
179 1 : Pattern.compile("(?i:someone|someone else|anonymous)|" + ACCOUNT_TEMPLATE_REGEX);
180 :
181 : /** Patterns to match change messages that need to be fixed. */
182 1 : private static final Pattern ASSIGNEE_DELETED_PATTERN = Pattern.compile("Assignee deleted: (.*)");
183 :
184 1 : private static final Pattern ASSIGNEE_ADDED_PATTERN = Pattern.compile("Assignee added: (.*)");
185 1 : private static final Pattern ASSIGNEE_CHANGED_PATTERN =
186 1 : Pattern.compile("Assignee changed from: (.*) to: (.*)");
187 :
188 1 : private static final Pattern REMOVED_REVIEWER_PATTERN =
189 1 : Pattern.compile(
190 : "Removed (cc|reviewer) (.*)(\\.| with the following votes:\n.*)", Pattern.DOTALL);
191 :
192 1 : private static final Pattern REMOVED_VOTE_PATTERN = Pattern.compile("Removed (.*) by (.*)");
193 :
194 : private static final String REMOVED_VOTES_CHANGE_MESSAGE_START = "Removed the following votes:";
195 1 : private static final Pattern REMOVED_VOTES_CHANGE_MESSAGE_PATTERN =
196 1 : Pattern.compile("\\* (.*) by (.*)");
197 :
198 1 : private static final Pattern REMOVED_CHANGE_MESSAGE_PATTERN =
199 1 : Pattern.compile("Change message removed by: (.*)(\nReason: .*)?");
200 :
201 1 : private static final Pattern SUBMITTED_PATTERN =
202 1 : Pattern.compile("Change has been successfully (.*) by (.*)");
203 :
204 1 : private static final Pattern ON_CODE_OWNER_ADD_REVIEWER_PATTERN =
205 1 : Pattern.compile("(.*) who was added as reviewer owns the following files");
206 :
207 : private static final String CODE_OWNER_ADD_REVIEWER_TAG =
208 : ChangeMessagesUtil.AUTOGENERATED_BY_GERRIT_TAG_PREFIX + "code-owners:addReviewer";
209 :
210 : private static final String ON_CODE_OWNER_APPROVAL_REGEX = "code-owner approved by (.*):";
211 : private static final String ON_CODE_OWNER_OVERRIDE_REGEX =
212 : "code-owners submit requirement .* overridden by (.*)";
213 :
214 1 : private static final Pattern ON_CODE_OWNER_REVIEW_PATTERN =
215 1 : Pattern.compile(ON_CODE_OWNER_APPROVAL_REGEX + "|" + ON_CODE_OWNER_OVERRIDE_REGEX);
216 1 : private static final Pattern ON_CODE_OWNER_POST_REVIEW_PATTERN =
217 1 : Pattern.compile("Patch Set [0-9]+:[\\s\\S]*By (voting|removing)[\\s\\S]*");
218 :
219 1 : private static final Pattern REPLY_BY_REASON_PATTERN =
220 1 : Pattern.compile("(.*) replied on the change");
221 1 : private static final Pattern ADDED_BY_REASON_PATTERN =
222 1 : Pattern.compile("Added by (.*) using the hovercard menu");
223 1 : private static final Pattern REMOVED_BY_REASON_PATTERN =
224 1 : Pattern.compile("Removed by (.*) using the hovercard menu");
225 1 : private static final Pattern REMOVED_BY_ICON_CLICK_REASON_PATTERN =
226 1 : Pattern.compile("Removed by (.*) by clicking the attention icon");
227 :
228 : /** Matches {@link Account#getNameEmail} */
229 1 : private static final Pattern NAME_EMAIL_PATTERN = Pattern.compile("(.*) (\\<.*\\>|\\(.*\\))");
230 :
231 1 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
232 :
233 1 : private static final Splitter COMMIT_MESSAGE_SPLITTER = Splitter.onPattern("\\r?\\n");
234 :
235 : private final ChangeNotes.Factory changeNotesFactory;
236 : private final AccountCache accountCache;
237 1 : private final DiffAlgorithm diffAlgorithm = new HistogramDiff();
238 1 : private static final Gson gson = OutputFormat.JSON_COMPACT.newGson();
239 :
240 : @Inject
241 1 : CommitRewriter(ChangeNotes.Factory changeNotesFactory, AccountCache accountCache) {
242 1 : this.changeNotesFactory = changeNotesFactory;
243 1 : this.accountCache = accountCache;
244 1 : }
245 :
246 : /**
247 : * Rewrites commit history of {@link RefNames#changeMetaRef}s in single {@code repo}. Only
248 : * rewrites branch if necessary, i.e. if there were any commits that contained user data.
249 : *
250 : * <p>See {@link RunOptions} for the execution and output options.
251 : *
252 : * @param project project to backfill
253 : * @param repo repo to backfill
254 : * @param options {@link RunOptions} to control how the run is executed.
255 : * @return BackfillResult
256 : */
257 : public BackfillResult backfillProject(
258 : Project.NameKey project, Repository repo, RunOptions options) {
259 :
260 1 : checkState(
261 : options.maxRefsInBatch > 0 && options.maxRefsToUpdate > 0,
262 : "Expected maxRefsInBatch>0 && <= maxRefsToUpdate>0");
263 1 : checkState(
264 : options.maxRefsInBatch <= options.maxRefsToUpdate,
265 : "Expected maxRefsInBatch(%s) <= maxRefsToUpdate(%s)",
266 : options.maxRefsInBatch,
267 : options.maxRefsToUpdate);
268 1 : BackfillResult result = new BackfillResult();
269 1 : result.ok = true;
270 1 : int refsInUpdate = 0;
271 :
272 : @SuppressWarnings("resource")
273 1 : RefsUpdate refsUpdate = null;
274 : try {
275 1 : for (Ref ref : repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES)) {
276 1 : if (result.fixedRefDiff.size() >= options.maxRefsToUpdate) {
277 1 : return result;
278 : }
279 1 : Change.Id changeId = Change.Id.fromRef(ref.getName());
280 1 : if (changeId == null || !ref.getName().equals(RefNames.changeMetaRef(changeId))) {
281 0 : continue;
282 : }
283 : try {
284 1 : ImmutableSet<AccountState> accountsInChange = ImmutableSet.of();
285 1 : if (options.verifyCommits) {
286 : try {
287 1 : ChangeNotes changeNotes = changeNotesFactory.create(project, changeId);
288 1 : accountsInChange = collectAccounts(changeNotes);
289 1 : } catch (Exception e) {
290 1 : logger.atWarning().withCause(e).log("Failed to run verification on ref %s", ref);
291 1 : }
292 : }
293 1 : if (refsUpdate == null) {
294 1 : refsUpdate = RefsUpdate.create(repo);
295 : }
296 1 : ChangeFixProgress changeFixProgress =
297 1 : backfillChange(refsUpdate, ref, accountsInChange, options);
298 1 : if (changeFixProgress.anyFixesApplied) {
299 1 : refsInUpdate++;
300 1 : refsUpdate
301 1 : .batchRefUpdate()
302 1 : .addCommand(
303 : new ReceiveCommand(
304 1 : ref.getObjectId(), changeFixProgress.newTipId, ref.getName()));
305 1 : result.fixedRefDiff.put(ref.getName(), changeFixProgress.commitDiffs);
306 : }
307 1 : if (refsInUpdate >= options.maxRefsInBatch
308 1 : || result.fixedRefDiff.size() >= options.maxRefsToUpdate) {
309 1 : processUpdate(options, refsUpdate);
310 1 : refsUpdate = null;
311 1 : refsInUpdate = 0;
312 : }
313 1 : if (!changeFixProgress.isValidAfterFix) {
314 1 : result.refsStillInvalidAfterFix.add(ref.getName());
315 : }
316 0 : } catch (Exception e) {
317 0 : logger.atWarning().withCause(e).log("Failed to fix ref %s", ref);
318 0 : result.refsFailedToFix.add(ref.getName());
319 1 : }
320 1 : }
321 1 : processUpdate(options, refsUpdate);
322 0 : } catch (IOException e) {
323 0 : logger.atWarning().log("Failed to fix project %s. Reason: %s", project.get(), e.getMessage());
324 0 : result.ok = false;
325 : } finally {
326 1 : if (refsUpdate != null) {
327 1 : refsUpdate.close();
328 : }
329 : }
330 :
331 1 : return result;
332 : }
333 :
334 : /** Executes a single {@link RefsUpdate#batchRefUpdate}. */
335 : private void processUpdate(RunOptions options, @Nullable RefsUpdate refsUpdate)
336 : throws IOException {
337 1 : if (refsUpdate == null) {
338 1 : return;
339 : }
340 1 : if (!refsUpdate.batchRefUpdate().getCommands().isEmpty()) {
341 1 : if (!options.dryRun) {
342 1 : refsUpdate.inserter().flush();
343 1 : RefUpdateUtil.executeChecked(refsUpdate.batchRefUpdate(), refsUpdate.revWalk());
344 : }
345 : }
346 1 : refsUpdate.close();
347 1 : }
348 :
349 : /**
350 : * Retrieves accounts, that are associated with a change (e.g. reviewers, commenters, etc.). These
351 : * accounts are used to verify that commits do not contain user data. See {@link #verifyCommit}
352 : *
353 : * @param changeNotes {@link ChangeNotes} of the change to retrieve associated accounts from.
354 : * @return {@link AccountState} of accounts, that are associated with the change.
355 : */
356 : private ImmutableSet<AccountState> collectAccounts(ChangeNotes changeNotes) {
357 1 : Set<Account.Id> accounts = new HashSet<>();
358 1 : accounts.add(changeNotes.getChange().getOwner());
359 1 : for (PatchSetApproval patchSetApproval : changeNotes.getApprovals().all().values()) {
360 1 : if (patchSetApproval.accountId() != null) {
361 1 : accounts.add(patchSetApproval.accountId());
362 : }
363 1 : if (patchSetApproval.realAccountId() != null) {
364 1 : accounts.add(patchSetApproval.realAccountId());
365 : }
366 1 : }
367 1 : accounts.addAll(changeNotes.getAllPastReviewers());
368 1 : accounts.addAll(changeNotes.getPastAssignees());
369 1 : changeNotes
370 1 : .getAttentionSetUpdates()
371 1 : .forEach(attentionSetUpdate -> accounts.add(attentionSetUpdate.account()));
372 1 : for (SubmitRecord submitRecord : changeNotes.getSubmitRecords()) {
373 1 : if (submitRecord.labels != null) {
374 1 : accounts.addAll(
375 1 : submitRecord.labels.stream()
376 1 : .map(label -> label.appliedBy)
377 1 : .filter(Objects::nonNull)
378 1 : .collect(Collectors.toSet()));
379 : }
380 1 : }
381 1 : for (HumanComment comment : changeNotes.getHumanComments().values()) {
382 0 : if (comment.author != null) {
383 0 : accounts.add(comment.author.getId());
384 : }
385 0 : if (comment.getRealAuthor() != null) {
386 0 : accounts.add(comment.getRealAuthor().getId());
387 : }
388 0 : }
389 1 : return ImmutableSet.copyOf(accountCache.get(accounts).values());
390 : }
391 :
392 : /** Verifies that the commit does not contain user data of accounts in {@code accounts}. */
393 : private boolean verifyCommit(
394 : String commitMessage, PersonIdent author, Collection<AccountState> accounts) {
395 1 : for (AccountState accountState : accounts) {
396 1 : Account account = accountState.account();
397 1 : if (commitMessage.contains(account.getName())) {
398 1 : return false;
399 : }
400 1 : if (account.fullName() != null && commitMessage.contains(account.fullName())) {
401 0 : return false;
402 : }
403 1 : if (account.displayName() != null && commitMessage.contains(account.displayName())) {
404 0 : return false;
405 : }
406 1 : if (account.preferredEmail() != null && commitMessage.contains(account.preferredEmail())) {
407 0 : return false;
408 : }
409 1 : if (accountState.userName().isPresent()
410 0 : && commitMessage.contains(accountState.userName().get())) {
411 0 : return false;
412 : }
413 1 : Stream<String> allEmails =
414 1 : accountState.externalIds().stream().map(ExternalId::email).filter(Objects::nonNull);
415 1 : if (allEmails.anyMatch(email -> commitMessage.contains(email))) {
416 0 : return false;
417 : }
418 1 : if (author.toString().contains(account.getName())) {
419 0 : return false;
420 : }
421 1 : }
422 1 : return true;
423 : }
424 :
425 : /**
426 : * Walks the ref history from oldest update to the most recent update, fixing the commits that
427 : * contain user data case by case. Commit history is rewritten from the first commit, that needs
428 : * to be updated, for all subsequent updates. The new ref tip is returned in {@link
429 : * ChangeFixProgress#newTipId}.
430 : */
431 : public ChangeFixProgress backfillChange(
432 : RefsUpdate refsUpdate,
433 : Ref ref,
434 : ImmutableSet<AccountState> accountsInChange,
435 : RunOptions options)
436 : throws IOException, ConfigInvalidException {
437 :
438 1 : ObjectId oldTip = ref.getObjectId();
439 : // Walk from the first commit of the branch.
440 1 : refsUpdate.revWalk().reset();
441 1 : refsUpdate.revWalk().markStart(refsUpdate.revWalk().parseCommit(oldTip));
442 1 : refsUpdate.revWalk().sort(RevSort.TOPO);
443 :
444 1 : refsUpdate.revWalk().sort(RevSort.REVERSE);
445 :
446 : RevCommit originalCommit;
447 :
448 1 : boolean rewriteStarted = false;
449 1 : ChangeFixProgress changeFixProgress = new ChangeFixProgress(ref.getName());
450 1 : while ((originalCommit = refsUpdate.revWalk().next()) != null) {
451 :
452 1 : changeFixProgress.updateAuthorId =
453 1 : parseIdent(changeFixProgress, originalCommit.getAuthorIdent());
454 : PersonIdent fixedAuthorIdent;
455 1 : if (changeFixProgress.updateAuthorId.isPresent()) {
456 1 : fixedAuthorIdent =
457 1 : getFixedIdent(originalCommit.getAuthorIdent(), changeFixProgress.updateAuthorId.get());
458 : } else {
459 : // Field to parse id from ident. Update by gerrit server or an old/broken change.
460 : // Leave as it is.
461 1 : fixedAuthorIdent = originalCommit.getAuthorIdent();
462 : }
463 1 : Optional<String> fixedCommitMessage = fixedCommitMessage(originalCommit, changeFixProgress);
464 : String commitMessage =
465 1 : fixedCommitMessage.isPresent()
466 1 : ? fixedCommitMessage.get()
467 1 : : originalCommit.getFullMessage();
468 1 : if (options.verifyCommits) {
469 1 : boolean isCommitValid = verifyCommit(commitMessage, fixedAuthorIdent, accountsInChange);
470 1 : changeFixProgress.isValidAfterFix &= isCommitValid;
471 1 : if (!isCommitValid) {
472 1 : StringBuilder detailedVerificationStatus =
473 : new StringBuilder(
474 1 : String.format(
475 : "Commit %s of ref %s failed verification after fix",
476 1 : originalCommit.getId(), ref));
477 1 : detailedVerificationStatus.append("\nCommit body:\n");
478 1 : detailedVerificationStatus.append(commitMessage);
479 1 : if (fixedCommitMessage.isPresent()) {
480 0 : detailedVerificationStatus.append("\n was fixed.\n");
481 : }
482 1 : detailedVerificationStatus.append("Commit author:\n");
483 1 : detailedVerificationStatus.append(fixedAuthorIdent.toString());
484 1 : logger.atWarning().log("%s", detailedVerificationStatus);
485 : }
486 : }
487 1 : boolean needsFix =
488 1 : !fixedAuthorIdent.equals(originalCommit.getAuthorIdent())
489 1 : || fixedCommitMessage.isPresent();
490 :
491 1 : if (!rewriteStarted && !needsFix) {
492 1 : changeFixProgress.newTipId = originalCommit;
493 1 : continue;
494 : }
495 1 : rewriteStarted = true;
496 1 : changeFixProgress.anyFixesApplied = true;
497 1 : CommitBuilder cb = new CommitBuilder();
498 1 : if (changeFixProgress.newTipId != null) {
499 1 : cb.setParentId(changeFixProgress.newTipId);
500 : }
501 1 : cb.setTreeId(originalCommit.getTree());
502 1 : cb.setMessage(commitMessage);
503 1 : cb.setAuthor(fixedAuthorIdent);
504 1 : cb.setCommitter(originalCommit.getCommitterIdent());
505 1 : cb.setEncoding(originalCommit.getEncoding());
506 1 : byte[] newCommitContent = cb.build();
507 1 : checkCommitModification(originalCommit, newCommitContent);
508 1 : changeFixProgress.newTipId =
509 1 : refsUpdate.inserter().insert(Constants.OBJ_COMMIT, newCommitContent);
510 : // Only compute diff if the content of the commit was actually changed.
511 1 : if (options.outputDiff && needsFix) {
512 1 : String diff = computeDiff(originalCommit.getRawBuffer(), newCommitContent);
513 1 : checkState(
514 1 : !Strings.isNullOrEmpty(diff),
515 : "Expected diff for commit %s of ref %s",
516 1 : originalCommit.getId(),
517 1 : ref.getName());
518 1 : changeFixProgress.commitDiffs.add(CommitDiff.create(originalCommit.getId(), diff));
519 1 : } else if (needsFix) {
520 : // Always output old commits SHA1
521 1 : changeFixProgress.commitDiffs.add(CommitDiff.create(originalCommit.getId(), ""));
522 : }
523 1 : }
524 1 : return changeFixProgress;
525 : }
526 :
527 : /**
528 : * In NoteDb, all the meta information is stored in footer lines. If we accidentally drop some of
529 : * the footer lines, the original meta information will be lost, and the change might become
530 : * unparsable.
531 : *
532 : * <p>While we can not verify the entire commit content, we at least make sure that the resulting
533 : * commit has the same author, committer and footer lines are in the same order and contain same
534 : * footer keys as the original commit.
535 : *
536 : * <p>Commit message and footer values might have been rewritten.
537 : */
538 : private void checkCommitModification(RevCommit originalCommit, byte[] newCommitContent)
539 : throws IOException {
540 1 : RevCommit newCommit = RevCommit.parse(newCommitContent);
541 1 : PersonIdent newAuthorIdent = newCommit.getAuthorIdent();
542 1 : PersonIdent originalAuthorIdent = originalCommit.getAuthorIdent();
543 : // The new commit must have same author and committer ident as the original commit.
544 1 : if (!verifyPersonIdent(newAuthorIdent, originalAuthorIdent)) {
545 0 : throw new IllegalStateException(
546 0 : String.format(
547 : "New author %s does not match original author %s",
548 0 : newAuthorIdent.toExternalString(), originalAuthorIdent.toExternalString()));
549 : }
550 1 : PersonIdent newCommitterIdent = newCommit.getCommitterIdent();
551 1 : PersonIdent originalCommitterIdent = originalCommit.getCommitterIdent();
552 1 : if (!verifyPersonIdent(newCommitterIdent, originalCommitterIdent)) {
553 0 : throw new IllegalStateException(
554 0 : String.format(
555 : "New committer %s does not match original committer %s",
556 0 : newCommitterIdent.toExternalString(), originalCommitterIdent.toExternalString()));
557 : }
558 :
559 1 : List<FooterLine> newFooterLines = newCommit.getFooterLines();
560 1 : List<FooterLine> originalFooterLines = originalCommit.getFooterLines();
561 : // Number and order of footer lines must remain the same, the value may have changed.
562 1 : if (newFooterLines.size() != originalFooterLines.size()) {
563 0 : String diff = computeDiff(originalCommit.getRawBuffer(), newCommitContent);
564 0 : throw new IllegalStateException(
565 0 : String.format(
566 : "Expected footer lines in new commit to match original footer lines. Diff %s", diff));
567 : }
568 1 : for (int i = 0; i < newFooterLines.size(); i++) {
569 1 : FooterLine newFooterLine = newFooterLines.get(i);
570 1 : FooterLine originalFooterLine = originalFooterLines.get(i);
571 1 : if (!newFooterLine.getKey().equals(originalFooterLine.getKey())) {
572 0 : String diff = computeDiff(originalCommit.getRawBuffer(), newCommitContent);
573 0 : throw new IllegalStateException(
574 0 : String.format(
575 : "Expected footer lines in new commit to match original footer lines. Diff %s",
576 : diff));
577 : }
578 : }
579 1 : }
580 :
581 : private boolean verifyPersonIdent(PersonIdent newIdent, PersonIdent originalIdent) {
582 1 : return newIdent.getTimeZoneOffset() == originalIdent.getTimeZoneOffset()
583 1 : && newIdent.getWhenAsInstant().equals(originalIdent.getWhenAsInstant())
584 1 : && newIdent.getEmailAddress().equals(originalIdent.getEmailAddress());
585 : }
586 :
587 : private Optional<String> fixAssigneeChangeMessage(
588 : ChangeFixProgress changeFixProgress,
589 : Optional<Account.Id> oldAssignee,
590 : Optional<Account.Id> newAssignee,
591 : String originalChangeMessage) {
592 1 : if (Strings.isNullOrEmpty(originalChangeMessage)) {
593 1 : return Optional.empty();
594 : }
595 :
596 1 : Matcher assigneeDeletedMatcher = ASSIGNEE_DELETED_PATTERN.matcher(originalChangeMessage);
597 1 : if (assigneeDeletedMatcher.matches()) {
598 1 : if (!NON_REPLACE_ACCOUNT_PATTERN.matcher(assigneeDeletedMatcher.group(1)).matches()) {
599 1 : Optional<String> assigneeReplacement =
600 1 : getPossibleAccountReplacement(
601 : changeFixProgress,
602 : oldAssignee,
603 1 : getAccountInfoFromNameEmail(assigneeDeletedMatcher.group(1)));
604 :
605 1 : return Optional.of(
606 1 : assigneeReplacement.isPresent()
607 1 : ? "Assignee deleted: " + assigneeReplacement.get()
608 0 : : "Assignee was deleted.");
609 : }
610 1 : return Optional.empty();
611 : }
612 :
613 1 : Matcher assigneeAddedMatcher = ASSIGNEE_ADDED_PATTERN.matcher(originalChangeMessage);
614 1 : if (assigneeAddedMatcher.matches()) {
615 1 : if (!NON_REPLACE_ACCOUNT_PATTERN.matcher(assigneeAddedMatcher.group(1)).matches()) {
616 1 : Optional<String> assigneeReplacement =
617 1 : getPossibleAccountReplacement(
618 : changeFixProgress,
619 : newAssignee,
620 1 : getAccountInfoFromNameEmail(assigneeAddedMatcher.group(1)));
621 1 : return Optional.of(
622 1 : assigneeReplacement.isPresent()
623 1 : ? "Assignee added: " + assigneeReplacement.get()
624 1 : : "Assignee was added.");
625 : }
626 1 : return Optional.empty();
627 : }
628 :
629 1 : Matcher assigneeChangedMatcher = ASSIGNEE_CHANGED_PATTERN.matcher(originalChangeMessage);
630 1 : if (assigneeChangedMatcher.matches()) {
631 1 : if (!NON_REPLACE_ACCOUNT_PATTERN.matcher(assigneeChangedMatcher.group(1)).matches()) {
632 1 : Optional<String> oldAssigneeReplacement =
633 1 : getPossibleAccountReplacement(
634 : changeFixProgress,
635 : oldAssignee,
636 1 : getAccountInfoFromNameEmail(assigneeChangedMatcher.group(1)));
637 1 : Optional<String> newAssigneeReplacement =
638 1 : getPossibleAccountReplacement(
639 : changeFixProgress,
640 : newAssignee,
641 1 : getAccountInfoFromNameEmail(assigneeChangedMatcher.group(2)));
642 1 : return Optional.of(
643 1 : oldAssigneeReplacement.isPresent() && newAssigneeReplacement.isPresent()
644 1 : ? String.format(
645 : "Assignee changed from: %s to: %s",
646 1 : oldAssigneeReplacement.get(), newAssigneeReplacement.get())
647 0 : : "Assignee was changed.");
648 : }
649 1 : return Optional.empty();
650 : }
651 1 : return Optional.empty();
652 : }
653 :
654 : private Optional<String> fixReviewerChangeMessage(String originalChangeMessage) {
655 1 : if (Strings.isNullOrEmpty(originalChangeMessage)) {
656 1 : return Optional.empty();
657 : }
658 1 : Matcher matcher = REMOVED_REVIEWER_PATTERN.matcher(originalChangeMessage);
659 :
660 1 : if (matcher.matches() && !ACCOUNT_TEMPLATE_PATTERN.matcher(matcher.group(2)).matches()) {
661 : // Since we do not use change messages for reviewer updates on UI, it does not matter what we
662 : // rewrite it to.
663 1 : return Optional.of(originalChangeMessage.substring(0, matcher.end(1)));
664 : }
665 1 : return Optional.empty();
666 : }
667 :
668 : private Optional<String> fixRemoveVoteChangeMessage(
669 : ChangeFixProgress changeFixProgress,
670 : Optional<Account.Id> reviewer,
671 : String originalChangeMessage) {
672 1 : if (Strings.isNullOrEmpty(originalChangeMessage)) {
673 1 : return Optional.empty();
674 : }
675 :
676 1 : Matcher matcher = REMOVED_VOTE_PATTERN.matcher(originalChangeMessage);
677 1 : if (matcher.matches() && !NON_REPLACE_ACCOUNT_PATTERN.matcher(matcher.group(2)).matches()) {
678 1 : Optional<String> reviewerReplacement =
679 1 : getPossibleAccountReplacement(
680 1 : changeFixProgress, reviewer, getAccountInfoFromNameEmail(matcher.group(2)));
681 1 : StringBuilder replacement = new StringBuilder();
682 1 : replacement.append("Removed ").append(matcher.group(1));
683 1 : if (reviewerReplacement.isPresent()) {
684 1 : replacement.append(" by ").append(reviewerReplacement.get());
685 : }
686 1 : return Optional.of(replacement.toString());
687 : }
688 1 : return Optional.empty();
689 : }
690 :
691 : private Optional<String> fixRemoveVotesChangeMessage(
692 : ChangeFixProgress changeFixProgress, String originalChangeMessage) {
693 1 : if (Strings.isNullOrEmpty(originalChangeMessage)
694 1 : || !originalChangeMessage.startsWith(REMOVED_VOTES_CHANGE_MESSAGE_START)) {
695 1 : return Optional.empty();
696 : }
697 1 : List<String> lines = COMMIT_MESSAGE_SPLITTER.splitToList(originalChangeMessage);
698 1 : StringBuilder fixedLines = new StringBuilder();
699 1 : boolean anyFixed = false;
700 1 : for (int i = 1; i < lines.size(); i++) {
701 1 : String line = lines.get(i);
702 1 : if (line.isEmpty()) {
703 0 : continue;
704 : }
705 1 : Matcher matcher = REMOVED_VOTES_CHANGE_MESSAGE_PATTERN.matcher(line);
706 1 : String replacementLine = line;
707 1 : if (matcher.matches() && !NON_REPLACE_ACCOUNT_PATTERN.matcher(matcher.group(2)).matches()) {
708 1 : anyFixed = true;
709 1 : Optional<String> reviewerReplacement =
710 1 : getPossibleAccountReplacement(
711 1 : changeFixProgress, Optional.empty(), getAccountInfoFromNameEmail(matcher.group(2)));
712 1 : replacementLine = "* " + matcher.group(1);
713 1 : if (reviewerReplacement.isPresent()) {
714 1 : replacementLine += " by " + reviewerReplacement.get();
715 : }
716 1 : replacementLine += "\n";
717 : }
718 1 : fixedLines.append(replacementLine);
719 : }
720 1 : if (!anyFixed) {
721 1 : return Optional.empty();
722 : }
723 1 : return Optional.of(REMOVED_VOTES_CHANGE_MESSAGE_START + "\n" + fixedLines);
724 : }
725 :
726 : private Optional<String> fixDeleteChangeMessageCommitMessage(String originalChangeMessage) {
727 1 : if (Strings.isNullOrEmpty(originalChangeMessage)) {
728 1 : return Optional.empty();
729 : }
730 :
731 1 : Matcher matcher = REMOVED_CHANGE_MESSAGE_PATTERN.matcher(originalChangeMessage);
732 1 : if (matcher.matches() && !ACCOUNT_TEMPLATE_PATTERN.matcher(matcher.group(1)).matches()) {
733 1 : String fixedMessage = "Change message removed";
734 1 : if (matcher.group(2) != null) {
735 1 : fixedMessage += matcher.group(2);
736 : }
737 1 : return Optional.of(fixedMessage);
738 : }
739 1 : return Optional.empty();
740 : }
741 :
742 : private Optional<String> fixSubmitChangeMessage(String originalChangeMessage) {
743 1 : if (Strings.isNullOrEmpty(originalChangeMessage)) {
744 1 : return Optional.empty();
745 : }
746 :
747 1 : Matcher matcher = SUBMITTED_PATTERN.matcher(originalChangeMessage);
748 1 : if (matcher.matches()) {
749 : // See https://gerrit-review.googlesource.com/c/gerrit/+/272654
750 1 : return Optional.of(originalChangeMessage.substring(0, matcher.end(1)));
751 : }
752 1 : return Optional.empty();
753 : }
754 :
755 : /**
756 : * Rewrites a code owners change message.
757 : *
758 : * <p>See https://gerrit-review.googlesource.com/c/plugins/code-owners/+/305409
759 : */
760 : private Optional<String> fixCodeOwnersOnAddReviewerChangeMessage(
761 : ChangeFixProgress changeFixProgress, String originalMessage) {
762 1 : if (Strings.isNullOrEmpty(originalMessage)) {
763 1 : return Optional.empty();
764 : }
765 :
766 1 : Matcher onAddReviewerMatcher = ON_CODE_OWNER_ADD_REVIEWER_PATTERN.matcher(originalMessage);
767 1 : if (!onAddReviewerMatcher.find()
768 : || NON_REPLACE_ACCOUNT_PATTERN
769 1 : .matcher(normalizeOnCodeOwnerAddReviewerMatch(onAddReviewerMatcher.group(1)))
770 1 : .matches()) {
771 1 : return Optional.empty();
772 : }
773 :
774 : // Pre fix, try to replace with something meaningful.
775 : // Retrieve reviewer accounts from cache and try to match by their name.
776 1 : onAddReviewerMatcher.reset();
777 1 : StringBuilder sb = new StringBuilder();
778 1 : while (onAddReviewerMatcher.find()) {
779 1 : String reviewerName = normalizeOnCodeOwnerAddReviewerMatch(onAddReviewerMatcher.group(1));
780 1 : Optional<String> replacementName =
781 1 : getPossibleAccountReplacement(
782 1 : changeFixProgress, Optional.empty(), ParsedAccountInfo.create(reviewerName));
783 1 : onAddReviewerMatcher.appendReplacement(
784 : sb,
785 1 : replacementName.isPresent()
786 1 : ? replacementName.get() + ", who was added as reviewer owns the following files"
787 1 : : "Added reviewer owns the following files");
788 1 : }
789 1 : onAddReviewerMatcher.appendTail(sb);
790 1 : sb.append("\n");
791 1 : return Optional.of(sb.toString());
792 : }
793 :
794 : /**
795 : * See {@link #ON_CODE_OWNER_ADD_REVIEWER_PATTERN}.
796 : *
797 : * <p>Some of the messages have format '{@link AccountTemplateUtil#ACCOUNT_TEMPLATE}, who...',
798 : * while others '{@link AccountTemplateUtil#ACCOUNT_TEMPLATE} who...'.
799 : *
800 : * <p>Cut the trailing ',' from the match, so that valid patterns are not replaced.
801 : */
802 : private static String normalizeOnCodeOwnerAddReviewerMatch(String reviewerMatch) {
803 1 : String reviewerName = reviewerMatch;
804 1 : if (reviewerName.charAt(reviewerName.length() - 1) == ',') {
805 1 : reviewerName = reviewerName.substring(0, reviewerName.length() - 1);
806 : }
807 1 : return reviewerName;
808 : }
809 :
810 : private Optional<String> fixCodeOwnersOnReviewChangeMessage(
811 : Optional<Account.Id> reviewer, String originalMessage) {
812 1 : if (Strings.isNullOrEmpty(originalMessage)) {
813 1 : return Optional.empty();
814 : }
815 1 : Matcher onCodeOwnerPostReviewMatcher =
816 1 : ON_CODE_OWNER_POST_REVIEW_PATTERN.matcher(originalMessage);
817 1 : if (!onCodeOwnerPostReviewMatcher.matches()) {
818 1 : return Optional.empty();
819 : }
820 1 : Matcher onCodeOwnerReviewMatcher = ON_CODE_OWNER_REVIEW_PATTERN.matcher(originalMessage);
821 1 : while (onCodeOwnerReviewMatcher.find()) {
822 1 : String accountName =
823 1 : firstNonNull(onCodeOwnerReviewMatcher.group(1), onCodeOwnerReviewMatcher.group(2));
824 1 : if (!ACCOUNT_TEMPLATE_PATTERN.matcher(accountName).matches()) {
825 1 : return Optional.of(
826 1 : originalMessage.replace(
827 : "by " + accountName,
828 : "by "
829 : + reviewer
830 1 : .map(AccountTemplateUtil::getAccountTemplate)
831 1 : .orElse(DEFAULT_ACCOUNT_REPLACEMENT))
832 : + "\n");
833 : }
834 1 : }
835 :
836 1 : return Optional.empty();
837 : }
838 :
839 : private Optional<String> fixAttentionSetReason(String originalReason) {
840 1 : if (Strings.isNullOrEmpty(originalReason)) {
841 0 : return Optional.empty();
842 : }
843 : // Only the latest attention set updates are displayed on UI. As long as reason is
844 : // human-readable, it does not matter what we rewrite it to.
845 :
846 1 : Matcher replyByReasonMatcher = REPLY_BY_REASON_PATTERN.matcher(originalReason);
847 1 : if (replyByReasonMatcher.matches()
848 1 : && !OK_ACCOUNT_NAME_PATTERN.matcher(replyByReasonMatcher.group(1)).matches()) {
849 1 : return Optional.of("Someone replied on the change");
850 : }
851 :
852 1 : Matcher addedByReasonMatcher = ADDED_BY_REASON_PATTERN.matcher(originalReason);
853 1 : if (addedByReasonMatcher.matches()
854 1 : && !OK_ACCOUNT_NAME_PATTERN.matcher(addedByReasonMatcher.group(1)).matches()) {
855 1 : return Optional.of("Added by someone using the hovercard menu");
856 : }
857 :
858 1 : Matcher removedByReasonMatcher = REMOVED_BY_REASON_PATTERN.matcher(originalReason);
859 1 : if (removedByReasonMatcher.matches()
860 1 : && !OK_ACCOUNT_NAME_PATTERN.matcher(removedByReasonMatcher.group(1)).matches()) {
861 :
862 1 : return Optional.of("Removed by someone using the hovercard menu");
863 : }
864 :
865 1 : Matcher removedByIconClickReasonMatcher =
866 1 : REMOVED_BY_ICON_CLICK_REASON_PATTERN.matcher(originalReason);
867 1 : if (removedByIconClickReasonMatcher.matches()
868 1 : && !OK_ACCOUNT_NAME_PATTERN.matcher(removedByIconClickReasonMatcher.group(1)).matches()) {
869 :
870 1 : return Optional.of("Removed by someone by clicking the attention icon");
871 : }
872 1 : return Optional.empty();
873 : }
874 :
875 : /**
876 : * Fixes commit body case by case, so it does not contain user data. Returns fixed commit message,
877 : * or {@link Optional#empty} if no fixes were applied.
878 : */
879 : private Optional<String> fixedCommitMessage(RevCommit revCommit, ChangeFixProgress fixProgress)
880 : throws ConfigInvalidException {
881 1 : byte[] raw = revCommit.getRawBuffer();
882 1 : Charset enc = RawParseUtils.parseEncoding(raw);
883 1 : Optional<CommitMessageRange> commitMessageRange =
884 1 : ChangeNoteUtil.parseCommitMessageRange(revCommit);
885 1 : if (!commitMessageRange.isPresent()) {
886 0 : throw new ConfigInvalidException("Failed to parse commit message " + revCommit.getName());
887 : }
888 1 : String changeSubject =
889 1 : RawParseUtils.decode(
890 : enc,
891 : raw,
892 1 : commitMessageRange.get().subjectStart(),
893 1 : commitMessageRange.get().subjectEnd());
894 1 : Optional<String> fixedChangeMessage = Optional.empty();
895 1 : String originalChangeMessage = null;
896 1 : if (commitMessageRange.get().hasChangeMessage()) {
897 1 : originalChangeMessage =
898 1 : RawParseUtils.decode(
899 : enc,
900 : raw,
901 1 : commitMessageRange.get().changeMessageStart(),
902 1 : commitMessageRange.get().changeMessageEnd() + 1)
903 1 : .trim();
904 : }
905 1 : List<FooterLine> footerLines = revCommit.getFooterLines();
906 1 : StringBuilder footerLinesBuilder = new StringBuilder();
907 1 : boolean anyFootersFixed = false;
908 1 : for (FooterLine fl : footerLines) {
909 1 : String footerKey = fl.getKey();
910 1 : String footerValue = fl.getValue();
911 1 : if (footerKey.equalsIgnoreCase(FOOTER_TAG.getName())) {
912 1 : fixProgress.tag = footerValue;
913 1 : } else if (footerKey.equalsIgnoreCase(FOOTER_ASSIGNEE.getName())) {
914 1 : Account.Id oldAssignee = fixProgress.assigneeId;
915 1 : FixIdentResult fixedAssignee = null;
916 1 : if (footerValue.equals("")) {
917 1 : fixProgress.assigneeId = null;
918 : } else {
919 1 : fixedAssignee = getFixedIdentString(fixProgress, footerValue);
920 1 : fixProgress.assigneeId = fixedAssignee.accountId;
921 : }
922 1 : if (!fixedChangeMessage.isPresent()) {
923 1 : fixedChangeMessage =
924 1 : fixAssigneeChangeMessage(
925 : fixProgress,
926 1 : Optional.ofNullable(oldAssignee),
927 1 : Optional.ofNullable(fixProgress.assigneeId),
928 : originalChangeMessage);
929 : }
930 1 : if (fixedAssignee != null && fixedAssignee.fixedIdentString.isPresent()) {
931 1 : addFooter(footerLinesBuilder, footerKey, fixedAssignee.fixedIdentString.get());
932 1 : anyFootersFixed = true;
933 1 : continue;
934 : }
935 1 : } else if (Arrays.stream(ReviewerStateInternal.values())
936 1 : .anyMatch(state -> footerKey.equalsIgnoreCase(state.getFooterKey().getName()))) {
937 1 : if (!fixedChangeMessage.isPresent()) {
938 1 : fixedChangeMessage = fixReviewerChangeMessage(originalChangeMessage);
939 : }
940 1 : FixIdentResult fixedReviewer = getFixedIdentString(fixProgress, footerValue);
941 1 : if (fixedReviewer.fixedIdentString.isPresent()) {
942 1 : addFooter(footerLinesBuilder, footerKey, fixedReviewer.fixedIdentString.get());
943 1 : anyFootersFixed = true;
944 1 : continue;
945 : }
946 1 : } else if (footerKey.equalsIgnoreCase(FOOTER_REAL_USER.getName())) {
947 1 : FixIdentResult fixedRealUser = getFixedIdentString(fixProgress, footerValue);
948 1 : if (fixedRealUser.fixedIdentString.isPresent()) {
949 1 : addFooter(footerLinesBuilder, footerKey, fixedRealUser.fixedIdentString.get());
950 1 : anyFootersFixed = true;
951 1 : continue;
952 : }
953 1 : } else if (footerKey.equalsIgnoreCase(FOOTER_LABEL.getName())) {
954 1 : int uuidStart = footerValue.indexOf(", ");
955 1 : int voterIdentStart = footerValue.indexOf(' ', uuidStart != -1 ? uuidStart + 2 : 0);
956 1 : FixIdentResult fixedVoter = null;
957 1 : if (voterIdentStart > 0) {
958 1 : String originalIdentString = footerValue.substring(voterIdentStart + 1);
959 1 : fixedVoter = getFixedIdentString(fixProgress, originalIdentString);
960 : }
961 1 : if (!fixedChangeMessage.isPresent()) {
962 1 : fixedChangeMessage =
963 1 : fixRemoveVoteChangeMessage(
964 : fixProgress,
965 1 : fixedVoter == null
966 1 : ? fixProgress.updateAuthorId
967 1 : : Optional.of(fixedVoter.accountId),
968 : originalChangeMessage);
969 : }
970 1 : if (fixedVoter != null && fixedVoter.fixedIdentString.isPresent()) {
971 1 : String fixedLabelVote =
972 1 : footerValue.substring(0, voterIdentStart) + " " + fixedVoter.fixedIdentString.get();
973 1 : addFooter(footerLinesBuilder, footerKey, fixedLabelVote);
974 1 : anyFootersFixed = true;
975 1 : continue;
976 : }
977 1 : } else if (footerKey.equalsIgnoreCase(FOOTER_SUBMITTED_WITH.getName())) {
978 : // Record format:
979 : // Submitted-with: OK
980 : // Submitted-with: OK: Code-Review: User Name <accountId@serverId>
981 1 : int voterIdentStart = StringUtils.ordinalIndexOf(footerValue, ": ", 2);
982 1 : if (voterIdentStart >= 0) {
983 1 : String originalIdentString = footerValue.substring(voterIdentStart + 2);
984 1 : FixIdentResult fixedVoter = getFixedIdentString(fixProgress, originalIdentString);
985 1 : if (fixedVoter.fixedIdentString.isPresent()) {
986 1 : String fixedLabelVote =
987 1 : footerValue.substring(0, voterIdentStart)
988 : + ": "
989 1 : + fixedVoter.fixedIdentString.get();
990 1 : addFooter(footerLinesBuilder, footerKey, fixedLabelVote);
991 1 : anyFootersFixed = true;
992 1 : continue;
993 : }
994 : }
995 :
996 1 : } else if (footerKey.equalsIgnoreCase(FOOTER_ATTENTION.getName())) {
997 1 : AttentionStatusInNoteDb originalAttentionSetUpdate =
998 1 : gson.fromJson(footerValue, AttentionStatusInNoteDb.class);
999 1 : FixIdentResult fixedAttentionAccount =
1000 1 : getFixedIdentString(fixProgress, originalAttentionSetUpdate.personIdent);
1001 1 : Optional<String> fixedReason = fixAttentionSetReason(originalAttentionSetUpdate.reason);
1002 1 : if (fixedAttentionAccount.fixedIdentString.isPresent() || fixedReason.isPresent()) {
1003 1 : AttentionStatusInNoteDb fixedAttentionSetUpdate =
1004 : new AttentionStatusInNoteDb(
1005 1 : fixedAttentionAccount.fixedIdentString.isPresent()
1006 1 : ? fixedAttentionAccount.fixedIdentString.get()
1007 1 : : originalAttentionSetUpdate.personIdent,
1008 : originalAttentionSetUpdate.operation,
1009 1 : fixedReason.isPresent() ? fixedReason.get() : originalAttentionSetUpdate.reason);
1010 1 : addFooter(footerLinesBuilder, footerKey, gson.toJson(fixedAttentionSetUpdate));
1011 1 : anyFootersFixed = true;
1012 1 : continue;
1013 : }
1014 : }
1015 1 : addFooter(footerLinesBuilder, footerKey, footerValue);
1016 1 : }
1017 : // Some of the old commits are missing corresponding footers but still have change messages that
1018 : // need the fix. For such cases, try to guess or replace with the default string (see
1019 : // getPossibleAccountReplacement)
1020 1 : if (!fixedChangeMessage.isPresent()) {
1021 1 : fixedChangeMessage = fixReviewerChangeMessage(originalChangeMessage);
1022 : }
1023 1 : if (!fixedChangeMessage.isPresent()) {
1024 1 : fixedChangeMessage = fixRemoveVotesChangeMessage(fixProgress, originalChangeMessage);
1025 : }
1026 1 : if (!fixedChangeMessage.isPresent()) {
1027 1 : fixedChangeMessage =
1028 1 : fixRemoveVoteChangeMessage(fixProgress, Optional.empty(), originalChangeMessage);
1029 : }
1030 1 : if (!fixedChangeMessage.isPresent()) {
1031 1 : fixedChangeMessage =
1032 1 : fixAssigneeChangeMessage(
1033 1 : fixProgress, Optional.empty(), Optional.empty(), originalChangeMessage);
1034 : }
1035 1 : if (!fixedChangeMessage.isPresent()) {
1036 1 : fixedChangeMessage = fixSubmitChangeMessage(originalChangeMessage);
1037 : }
1038 1 : if (!fixedChangeMessage.isPresent()) {
1039 1 : fixedChangeMessage = fixDeleteChangeMessageCommitMessage(originalChangeMessage);
1040 : }
1041 1 : if (!fixedChangeMessage.isPresent()) {
1042 1 : fixedChangeMessage =
1043 1 : fixCodeOwnersOnReviewChangeMessage(fixProgress.updateAuthorId, originalChangeMessage);
1044 : }
1045 1 : if (!fixedChangeMessage.isPresent()
1046 1 : && Objects.equals(fixProgress.tag, CODE_OWNER_ADD_REVIEWER_TAG)) {
1047 1 : fixedChangeMessage =
1048 1 : fixCodeOwnersOnAddReviewerChangeMessage(fixProgress, originalChangeMessage);
1049 : }
1050 1 : if (!anyFootersFixed && !fixedChangeMessage.isPresent()) {
1051 1 : return Optional.empty();
1052 : }
1053 1 : StringBuilder fixedCommitBuilder = new StringBuilder();
1054 1 : fixedCommitBuilder.append(changeSubject);
1055 1 : fixedCommitBuilder.append("\n\n");
1056 1 : if (commitMessageRange.get().hasChangeMessage()) {
1057 1 : fixedCommitBuilder.append(fixedChangeMessage.orElse(originalChangeMessage));
1058 1 : fixedCommitBuilder.append("\n\n");
1059 : }
1060 1 : fixedCommitBuilder.append(footerLinesBuilder);
1061 1 : return Optional.of(fixedCommitBuilder.toString());
1062 : }
1063 :
1064 : private static StringBuilder addFooter(StringBuilder sb, String footer, String value) {
1065 1 : if (value == null) {
1066 0 : return sb;
1067 : }
1068 1 : sb.append(footer).append(":");
1069 1 : sb.append(" ").append(value);
1070 1 : sb.append('\n');
1071 1 : return sb;
1072 : }
1073 :
1074 : private Optional<Account.Id> parseIdent(ChangeFixProgress changeFixProgress, PersonIdent ident) {
1075 1 : Optional<Account.Id> account = NoteDbUtil.parseIdent(ident);
1076 1 : if (account.isPresent()) {
1077 1 : changeFixProgress.parsedAccounts.putIfAbsent(account.get(), Optional.empty());
1078 : } else {
1079 1 : logger.atWarning().log(
1080 : "Fixing ref %s, failed to parse id %s", changeFixProgress.changeMetaRef, ident);
1081 : }
1082 1 : return account;
1083 : }
1084 :
1085 : /**
1086 : * Fixes {@code originalIdent} so it does not contain user data, see {@link
1087 : * ChangeNoteUtil#getAccountIdAsUsername}.
1088 : */
1089 : private PersonIdent getFixedIdent(PersonIdent originalIdent, Account.Id identAccount) {
1090 1 : return new PersonIdent(
1091 1 : ChangeNoteUtil.getAccountIdAsUsername(identAccount),
1092 1 : originalIdent.getEmailAddress(),
1093 1 : originalIdent.getWhen(),
1094 1 : originalIdent.getTimeZone());
1095 : }
1096 :
1097 : /**
1098 : * Parses {@code originalIdentString} and applies the fix, so it does not contain user data, see
1099 : * {@link ChangeNoteUtil#appendAccountIdIdentString}.
1100 : *
1101 : * @param changeFixProgress see {@link ChangeFixProgress}
1102 : * @param originalIdentString ident to apply the fix to.
1103 : * @return {@link FixIdentResult}, with {@link FixIdentResult#accountId} parsed from {@code
1104 : * originalIdentString} and {@link FixIdentResult#fixedIdentString} if the fix was applied.
1105 : * @throws ConfigInvalidException if could not parse {@link FixIdentResult#accountId} from {@code
1106 : * originalIdentString}
1107 : */
1108 : private FixIdentResult getFixedIdentString(
1109 : ChangeFixProgress changeFixProgress, String originalIdentString)
1110 : throws ConfigInvalidException {
1111 1 : FixIdentResult fixIdentResult = new FixIdentResult();
1112 1 : PersonIdent originalIdent = RawParseUtils.parsePersonIdent(originalIdentString);
1113 : // Ident as String is saved in NoteDB footers, if this fails to parse, something is
1114 : // wrong with the change and we better not touch it.
1115 1 : fixIdentResult.accountId =
1116 1 : parseIdent(changeFixProgress, originalIdent)
1117 1 : .orElseThrow(
1118 0 : () -> new ConfigInvalidException("field to parse id: " + originalIdentString));
1119 1 : String fixedIdentString =
1120 1 : ChangeNoteUtil.formatAccountIdentString(
1121 1 : fixIdentResult.accountId, originalIdent.getEmailAddress());
1122 1 : fixIdentResult.fixedIdentString =
1123 1 : fixedIdentString.equals(originalIdentString)
1124 1 : ? Optional.empty()
1125 1 : : Optional.of(fixedIdentString);
1126 1 : return fixIdentResult;
1127 : }
1128 :
1129 : /** Extracts {@link ParsedAccountInfo} from {@link Account#getNameEmail} */
1130 : private ParsedAccountInfo getAccountInfoFromNameEmail(String nameEmail) {
1131 1 : Matcher nameEmailMatcher = NAME_EMAIL_PATTERN.matcher(nameEmail);
1132 1 : if (!nameEmailMatcher.matches()) {
1133 1 : return ParsedAccountInfo.create(nameEmail);
1134 : }
1135 :
1136 1 : return ParsedAccountInfo.create(
1137 1 : nameEmailMatcher.group(1),
1138 1 : nameEmailMatcher.group(2).substring(1, nameEmailMatcher.group(2).length() - 1));
1139 : }
1140 :
1141 : /**
1142 : * Returns replacement for {@code accountName}.
1143 : *
1144 : * <p>If {@code account} is known, replace with {@link AccountTemplateUtil#getAccountTemplate}.
1145 : * Otherwise, try to guess the correct replacement account for {@code accountName} among {@link
1146 : * ChangeFixProgress#parsedAccounts} that appeared in the change. If this fails {@link
1147 : * Optional#empty} is returned.
1148 : *
1149 : * @param changeFixProgress see {@link ChangeFixProgress}
1150 : * @param account account that should be used for replacement, if known
1151 : * @param accountInfo {@link ParsedAccountInfo} to replace.
1152 : * @return replacement for {@code accountName} or {@link Optional#empty}, if the replacement could
1153 : * not be determined.
1154 : */
1155 : private Optional<String> getPossibleAccountReplacement(
1156 : ChangeFixProgress changeFixProgress,
1157 : Optional<Account.Id> account,
1158 : ParsedAccountInfo accountInfo) {
1159 1 : if (account.isPresent()) {
1160 1 : return Optional.of(AccountTemplateUtil.getAccountTemplate(account.get()));
1161 : }
1162 : // Retrieve reviewer accounts from cache and try to match by their name.
1163 1 : Map<Account.Id, AccountState> missingAccountStateReviewers =
1164 1 : accountCache.get(
1165 1 : changeFixProgress.parsedAccounts.entrySet().stream()
1166 1 : .filter(entry -> !entry.getValue().isPresent())
1167 1 : .map(Map.Entry::getKey)
1168 1 : .collect(ImmutableSet.toImmutableSet()));
1169 1 : changeFixProgress.parsedAccounts.putAll(
1170 1 : missingAccountStateReviewers.entrySet().stream()
1171 1 : .collect(
1172 1 : ImmutableMap.toImmutableMap(
1173 1 : Map.Entry::getKey, e -> Optional.ofNullable(e.getValue()))));
1174 1 : Map<Account.Id, AccountState> possibleReplacements = ImmutableMap.of();
1175 1 : if (accountInfo.email().isPresent()) {
1176 1 : possibleReplacements =
1177 1 : changeFixProgress.parsedAccounts.entrySet().stream()
1178 1 : .filter(
1179 : e ->
1180 1 : e.getValue().isPresent()
1181 1 : && Objects.equals(
1182 1 : e.getValue().get().account().preferredEmail(),
1183 1 : accountInfo.email().get()))
1184 1 : .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, e -> e.getValue().get()));
1185 : // Filter further so we match both email & name
1186 1 : if (possibleReplacements.size() > 1) {
1187 0 : logger.atWarning().log(
1188 : "Fixing ref %s, multiple accounts found with the same email address, while replacing"
1189 : + " %s",
1190 : changeFixProgress.changeMetaRef, accountInfo);
1191 0 : possibleReplacements =
1192 0 : possibleReplacements.entrySet().stream()
1193 0 : .filter(e -> Objects.equals(e.getValue().account().getName(), accountInfo.name()))
1194 0 : .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
1195 : }
1196 : }
1197 1 : if (possibleReplacements.isEmpty()) {
1198 1 : possibleReplacements =
1199 1 : changeFixProgress.parsedAccounts.entrySet().stream()
1200 1 : .filter(
1201 : e ->
1202 1 : e.getValue().isPresent()
1203 1 : && Objects.equals(
1204 1 : e.getValue().get().account().getName(), accountInfo.name()))
1205 1 : .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, e -> e.getValue().get()));
1206 : }
1207 1 : Optional<String> replacementName = Optional.empty();
1208 1 : if (possibleReplacements.isEmpty()) {
1209 1 : logger.atWarning().log(
1210 : "Fixing ref %s, could not find reviewer account matching name %s",
1211 : changeFixProgress.changeMetaRef, accountInfo);
1212 1 : } else if (possibleReplacements.size() > 1) {
1213 1 : logger.atWarning().log(
1214 : "Fixing ref %s found multiple reviewer account matching name %s",
1215 : changeFixProgress.changeMetaRef, accountInfo);
1216 : } else {
1217 1 : replacementName =
1218 1 : Optional.of(
1219 1 : AccountTemplateUtil.getAccountTemplate(
1220 1 : Iterables.getOnlyElement(possibleReplacements.keySet())));
1221 : }
1222 1 : return replacementName;
1223 : }
1224 :
1225 : /**
1226 : * Cuts tree and parent lines from raw unparsed commit body, so they are not included in diff
1227 : * comparison.
1228 : *
1229 : * @param b raw unparsed commit body, see {@link RevCommit#getRawBuffer()}.
1230 : * <p>For parsing, see {@link RawParseUtils#author}, {@link RawParseUtils#commitMessage}, etc.
1231 : * @return raw unparsed commit body, without tree and parent lines.
1232 : */
1233 : public static byte[] cutTreeAndParents(byte[] b) {
1234 1 : final int sz = b.length;
1235 1 : int ptr = 46; // skip the "tree ..." line.
1236 1 : while (ptr < sz && b[ptr] == 'p') {
1237 1 : ptr += 48;
1238 : } // skip this parent.
1239 1 : return Arrays.copyOfRange(b, ptr, b.length + 1);
1240 : }
1241 :
1242 : private String computeDiff(byte[] oldCommit, byte[] newCommit) throws IOException {
1243 1 : RawText oldBody = new RawText(cutTreeAndParents(oldCommit));
1244 1 : RawText newBody = new RawText(cutTreeAndParents(newCommit));
1245 1 : ByteArrayOutputStream out = new ByteArrayOutputStream();
1246 1 : EditList diff = diffAlgorithm.diff(RawTextComparator.DEFAULT, oldBody, newBody);
1247 1 : try (DiffFormatter fmt = new DiffFormatter(out)) {
1248 : // Do not show any unchanged lines, since it is not interesting
1249 1 : fmt.setContext(0);
1250 1 : fmt.format(diff, oldBody, newBody);
1251 1 : fmt.flush();
1252 1 : return out.toString(UTF_8);
1253 : }
1254 : }
1255 :
1256 : private static ObjectInserter newPackInserter(Repository repo) {
1257 1 : if (!(repo instanceof FileRepository)) {
1258 1 : return repo.newObjectInserter();
1259 : }
1260 0 : PackInserter ins = ((FileRepository) repo).getObjectDatabase().newPackInserter();
1261 0 : ins.checkExisting(false);
1262 0 : return ins;
1263 : }
1264 :
1265 : /**
1266 : * Parsed and fixed {@link PersonIdent} string, formatted as {@link
1267 : * ChangeNoteUtil#appendAccountIdIdentString}
1268 : */
1269 : private static class FixIdentResult {
1270 :
1271 : /** {@link com.google.gerrit.entities.Account.Id} parsed from PersonIdent string. */
1272 : Account.Id accountId;
1273 : /**
1274 : * Fixed ident string, that does not contain user data, or {@link Optional#empty} if fix was not
1275 : * required.
1276 : */
1277 : Optional<String> fixedIdentString;
1278 : }
1279 :
1280 : /**
1281 : * Holds the state of change rewrite progress. Rewrite goes from the oldest commit to the most
1282 : * recent update.
1283 : */
1284 : private static class ChangeFixProgress {
1285 :
1286 : /** {@link RefNames#changeMetaRef} of the change that is being fixed. */
1287 : final String changeMetaRef;
1288 :
1289 : /** Tag at current commit update. */
1290 1 : String tag = null;
1291 :
1292 : /** Assignee at current commit update. */
1293 1 : Account.Id assigneeId = null;
1294 :
1295 : /** Author of the current commit update. */
1296 1 : Optional<Account.Id> updateAuthorId = null;
1297 :
1298 : /**
1299 : * Accounts parsed so far together with their {@link Account#getName} extracted from {@link
1300 : * #accountCache} if needed by rewrite. Maps to empty string if was not requested from cache
1301 : * yet.
1302 : */
1303 1 : Map<Account.Id, Optional<AccountState>> parsedAccounts = new HashMap<>();
1304 :
1305 : /** Id of the current commit in rewriter walk. */
1306 1 : ObjectId newTipId = null;
1307 : /** If any commits were rewritten by the rewriter. */
1308 1 : boolean anyFixesApplied = false;
1309 :
1310 : /**
1311 : * Whether all commits seen by the rewriter with the fixes applied passed the verification, see
1312 : * {@link #verifyCommit}.
1313 : */
1314 1 : boolean isValidAfterFix = true;
1315 :
1316 1 : List<CommitDiff> commitDiffs = new ArrayList<>();
1317 :
1318 1 : public ChangeFixProgress(String changeMetaRef) {
1319 1 : this.changeMetaRef = changeMetaRef;
1320 1 : }
1321 : }
1322 :
1323 : /**
1324 : * Account info parsed from {@link Account#getNameEmail}. See {@link
1325 : * #getAccountInfoFromNameEmail}.
1326 : */
1327 : @AutoValue
1328 1 : abstract static class ParsedAccountInfo {
1329 :
1330 : static ParsedAccountInfo create(String fullName, String email) {
1331 1 : return new AutoValue_CommitRewriter_ParsedAccountInfo(fullName, Optional.ofNullable(email));
1332 : }
1333 :
1334 : static ParsedAccountInfo create(String fullName) {
1335 1 : return new AutoValue_CommitRewriter_ParsedAccountInfo(fullName, Optional.empty());
1336 : }
1337 :
1338 : abstract String name();
1339 :
1340 : abstract Optional<String> email();
1341 : }
1342 :
1343 : /**
1344 : * Objects, needed to fix Refs in a single {@link BatchRefUpdate}. Number of changes in a batch
1345 : * are limited by {@link RunOptions#maxRefsInBatch}.
1346 : */
1347 : @AutoValue
1348 1 : abstract static class RefsUpdate implements AutoCloseable {
1349 : static RefsUpdate create(Repository repo) {
1350 1 : RevWalk revWalk = new RevWalk(repo);
1351 1 : ObjectInserter inserter = newPackInserter(repo);
1352 1 : BatchRefUpdate bru = repo.getRefDatabase().newBatchUpdate();
1353 1 : bru.setForceRefLog(true);
1354 1 : bru.setRefLogMessage(CommitRewriter.class.getName(), false);
1355 1 : bru.setAllowNonFastForwards(true);
1356 1 : return new AutoValue_CommitRewriter_RefsUpdate(bru, revWalk, inserter);
1357 : }
1358 :
1359 : @Override
1360 : public void close() {
1361 1 : inserter().close();
1362 1 : revWalk().close();
1363 1 : }
1364 :
1365 : abstract BatchRefUpdate batchRefUpdate();
1366 :
1367 : abstract RevWalk revWalk();
1368 :
1369 : abstract ObjectInserter inserter();
1370 : }
1371 : }
|
