Line data Source code
1 : // Copyright (C) 2020 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.patch;
16 :
17 : import static com.google.gerrit.entities.Patch.COMMIT_MSG;
18 : import static com.google.gerrit.entities.Patch.MERGE_LIST;
19 :
20 : import com.google.auto.value.AutoValue;
21 : import com.google.common.collect.ImmutableCollection;
22 : import com.google.common.collect.ImmutableList;
23 : import com.google.common.collect.ImmutableMap;
24 : import com.google.common.flogger.FluentLogger;
25 : import com.google.gerrit.common.Nullable;
26 : import com.google.gerrit.entities.Patch;
27 : import com.google.gerrit.entities.Patch.ChangeType;
28 : import com.google.gerrit.entities.Project;
29 : import com.google.gerrit.extensions.client.DiffPreferencesInfo;
30 : import com.google.gerrit.extensions.client.DiffPreferencesInfo.Whitespace;
31 : import com.google.gerrit.server.cache.CacheModule;
32 : import com.google.gerrit.server.patch.diff.ModifiedFilesCache;
33 : import com.google.gerrit.server.patch.diff.ModifiedFilesCacheImpl;
34 : import com.google.gerrit.server.patch.diff.ModifiedFilesCacheKey;
35 : import com.google.gerrit.server.patch.filediff.FileDiffCache;
36 : import com.google.gerrit.server.patch.filediff.FileDiffCacheImpl;
37 : import com.google.gerrit.server.patch.filediff.FileDiffCacheKey;
38 : import com.google.gerrit.server.patch.filediff.FileDiffOutput;
39 : import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheImpl;
40 : import com.google.gerrit.server.patch.gitdiff.ModifiedFile;
41 : import com.google.gerrit.server.patch.gitfilediff.GitFileDiffCacheImpl;
42 : import com.google.gerrit.server.patch.gitfilediff.GitFileDiffCacheImpl.DiffAlgorithm;
43 : import com.google.inject.Inject;
44 : import com.google.inject.Module;
45 : import com.google.inject.Singleton;
46 : import java.io.IOException;
47 : import java.util.ArrayList;
48 : import java.util.List;
49 : import java.util.Map;
50 : import java.util.Optional;
51 : import java.util.function.Function;
52 : import java.util.stream.Collectors;
53 : import org.eclipse.jgit.diff.DiffEntry;
54 : import org.eclipse.jgit.diff.DiffFormatter;
55 : import org.eclipse.jgit.lib.Config;
56 : import org.eclipse.jgit.lib.ObjectId;
57 : import org.eclipse.jgit.lib.ObjectReader;
58 : import org.eclipse.jgit.revwalk.RevWalk;
59 : import org.eclipse.jgit.util.io.DisabledOutputStream;
60 :
61 : /**
62 : * Provides different file diff operations. Uses the underlying Git/Gerrit caches to speed up the
63 : * diff computation.
64 : */
65 : @Singleton
66 : public class DiffOperationsImpl implements DiffOperations {
67 152 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
68 :
69 152 : private static final ImmutableMap<DiffEntry.ChangeType, Patch.ChangeType> changeTypeMap =
70 152 : ImmutableMap.of(
71 : DiffEntry.ChangeType.ADD,
72 : Patch.ChangeType.ADDED,
73 : DiffEntry.ChangeType.MODIFY,
74 : Patch.ChangeType.MODIFIED,
75 : DiffEntry.ChangeType.DELETE,
76 : Patch.ChangeType.DELETED,
77 : DiffEntry.ChangeType.RENAME,
78 : Patch.ChangeType.RENAMED,
79 : DiffEntry.ChangeType.COPY,
80 : Patch.ChangeType.COPIED);
81 :
82 : private static final int RENAME_SCORE = 60;
83 152 : private static final DiffAlgorithm DEFAULT_DIFF_ALGORITHM =
84 : DiffAlgorithm.HISTOGRAM_WITH_FALLBACK_MYERS;
85 152 : private static final Whitespace DEFAULT_WHITESPACE = Whitespace.IGNORE_NONE;
86 :
87 : private final ModifiedFilesCache modifiedFilesCache;
88 : private final FileDiffCache fileDiffCache;
89 : private final BaseCommitUtil baseCommitUtil;
90 :
91 : public static Module module() {
92 152 : return new CacheModule() {
93 : @Override
94 : protected void configure() {
95 152 : bind(DiffOperations.class).to(DiffOperationsImpl.class);
96 152 : install(GitModifiedFilesCacheImpl.module());
97 152 : install(ModifiedFilesCacheImpl.module());
98 152 : install(GitFileDiffCacheImpl.module());
99 152 : install(FileDiffCacheImpl.module());
100 152 : }
101 : };
102 : }
103 :
104 : @Inject
105 : public DiffOperationsImpl(
106 : ModifiedFilesCache modifiedFilesCache,
107 : FileDiffCache fileDiffCache,
108 152 : BaseCommitUtil baseCommit) {
109 152 : this.modifiedFilesCache = modifiedFilesCache;
110 152 : this.fileDiffCache = fileDiffCache;
111 152 : this.baseCommitUtil = baseCommit;
112 152 : }
113 :
114 : @Override
115 : public Map<String, FileDiffOutput> listModifiedFilesAgainstParent(
116 : Project.NameKey project, ObjectId newCommit, int parent, DiffOptions diffOptions)
117 : throws DiffNotAvailableException {
118 : try {
119 104 : DiffParameters diffParams = computeDiffParameters(project, newCommit, parent);
120 104 : return getModifiedFiles(diffParams, diffOptions);
121 0 : } catch (IOException e) {
122 0 : throw new DiffNotAvailableException(
123 : "Failed to evaluate the parent/base commit for commit " + newCommit, e);
124 : }
125 : }
126 :
127 : @Override
128 : public Map<String, ModifiedFile> loadModifiedFilesAgainstParent(
129 : Project.NameKey project,
130 : ObjectId newCommit,
131 : int parentNum,
132 : DiffOptions diffOptions,
133 : RevWalk revWalk,
134 : Config repoConfig)
135 : throws DiffNotAvailableException {
136 : try {
137 3 : DiffParameters diffParams = computeDiffParameters(project, newCommit, parentNum);
138 3 : return loadModifiedFilesWithoutCache(project, diffParams, revWalk, repoConfig);
139 0 : } catch (IOException e) {
140 0 : throw new DiffNotAvailableException(
141 0 : String.format(
142 : "Failed to evaluate the parent/base commit for commit '%s' with parentNum=%d",
143 0 : newCommit, parentNum),
144 : e);
145 : }
146 : }
147 :
148 : @Override
149 : public Map<String, FileDiffOutput> listModifiedFiles(
150 : Project.NameKey project, ObjectId oldCommit, ObjectId newCommit, DiffOptions diffOptions)
151 : throws DiffNotAvailableException {
152 : DiffParameters params =
153 10 : DiffParameters.builder()
154 10 : .project(project)
155 10 : .newCommit(newCommit)
156 10 : .baseCommit(oldCommit)
157 10 : .comparisonType(ComparisonType.againstOtherPatchSet())
158 10 : .build();
159 10 : return getModifiedFiles(params, diffOptions);
160 : }
161 :
162 : @Override
163 : public Map<String, ModifiedFile> loadModifiedFiles(
164 : Project.NameKey project,
165 : ObjectId oldCommit,
166 : ObjectId newCommit,
167 : DiffOptions diffOptions,
168 : RevWalk revWalk,
169 : Config repoConfig)
170 : throws DiffNotAvailableException {
171 : DiffParameters params =
172 3 : DiffParameters.builder()
173 3 : .project(project)
174 3 : .newCommit(newCommit)
175 3 : .baseCommit(oldCommit)
176 3 : .comparisonType(ComparisonType.againstOtherPatchSet())
177 3 : .build();
178 3 : return loadModifiedFilesWithoutCache(project, params, revWalk, repoConfig);
179 : }
180 :
181 : @Override
182 : public FileDiffOutput getModifiedFileAgainstParent(
183 : Project.NameKey project,
184 : ObjectId newCommit,
185 : int parent,
186 : String fileName,
187 : @Nullable DiffPreferencesInfo.Whitespace whitespace)
188 : throws DiffNotAvailableException {
189 : try {
190 8 : DiffParameters diffParams = computeDiffParameters(project, newCommit, parent);
191 8 : FileDiffCacheKey key =
192 8 : createFileDiffCacheKey(
193 : project,
194 8 : diffParams.baseCommit(),
195 : newCommit,
196 : fileName,
197 : DEFAULT_DIFF_ALGORITHM,
198 : /* useTimeout= */ true,
199 : whitespace);
200 8 : return getModifiedFileForKey(key);
201 0 : } catch (IOException e) {
202 0 : throw new DiffNotAvailableException(
203 : "Failed to evaluate the parent/base commit for commit " + newCommit, e);
204 : }
205 : }
206 :
207 : @Override
208 : public FileDiffOutput getModifiedFile(
209 : Project.NameKey project,
210 : ObjectId oldCommit,
211 : ObjectId newCommit,
212 : String fileName,
213 : @Nullable DiffPreferencesInfo.Whitespace whitespace)
214 : throws DiffNotAvailableException {
215 7 : FileDiffCacheKey key =
216 7 : createFileDiffCacheKey(
217 : project,
218 : oldCommit,
219 : newCommit,
220 : fileName,
221 : DEFAULT_DIFF_ALGORITHM,
222 : /* useTimeout= */ true,
223 : whitespace);
224 7 : return getModifiedFileForKey(key);
225 : }
226 :
227 : private ImmutableMap<String, FileDiffOutput> getModifiedFiles(
228 : DiffParameters diffParams, DiffOptions diffOptions) throws DiffNotAvailableException {
229 : try {
230 104 : Project.NameKey project = diffParams.project();
231 104 : ObjectId newCommit = diffParams.newCommit();
232 104 : ObjectId oldCommit = diffParams.baseCommit();
233 104 : ComparisonType cmp = diffParams.comparisonType();
234 :
235 104 : ImmutableList<ModifiedFile> modifiedFiles =
236 104 : modifiedFilesCache.get(createModifiedFilesKey(project, oldCommit, newCommit));
237 :
238 104 : List<FileDiffCacheKey> fileCacheKeys = new ArrayList<>();
239 104 : fileCacheKeys.add(
240 104 : createFileDiffCacheKey(
241 : project,
242 : oldCommit,
243 : newCommit,
244 : COMMIT_MSG,
245 : DEFAULT_DIFF_ALGORITHM,
246 : /* useTimeout= */ true,
247 : /* whitespace= */ null));
248 :
249 104 : if (cmp.isAgainstAutoMerge() || isMergeAgainstParent(cmp, project, newCommit)) {
250 31 : fileCacheKeys.add(
251 31 : createFileDiffCacheKey(
252 : project,
253 : oldCommit,
254 : newCommit,
255 : MERGE_LIST,
256 : DEFAULT_DIFF_ALGORITHM,
257 : /* useTimeout= */ true,
258 : /*whitespace = */ null));
259 : }
260 :
261 104 : if (diffParams.skipFiles() == null) {
262 104 : modifiedFiles.stream()
263 104 : .map(
264 : entity ->
265 93 : createFileDiffCacheKey(
266 : project,
267 : oldCommit,
268 : newCommit,
269 93 : entity.newPath().isPresent()
270 93 : ? entity.newPath().get()
271 93 : : entity.oldPath().get(),
272 : DEFAULT_DIFF_ALGORITHM,
273 : /* useTimeout= */ true,
274 : /* whitespace= */ null))
275 104 : .forEach(fileCacheKeys::add);
276 : }
277 104 : return getModifiedFilesForKeys(fileCacheKeys, diffOptions);
278 0 : } catch (IOException e) {
279 0 : throw new DiffNotAvailableException(e);
280 : }
281 : }
282 :
283 : private FileDiffOutput getModifiedFileForKey(FileDiffCacheKey key)
284 : throws DiffNotAvailableException {
285 13 : Map<String, FileDiffOutput> diffList =
286 13 : getModifiedFilesForKeys(ImmutableList.of(key), DiffOptions.DEFAULTS);
287 13 : return diffList.containsKey(key.newFilePath())
288 13 : ? diffList.get(key.newFilePath())
289 2 : : FileDiffOutput.empty(key.newFilePath(), key.oldCommit(), key.newCommit());
290 : }
291 :
292 : /**
293 : * Lookup the file diffs for the input {@code keys}. For results where the cache reports negative
294 : * results, e.g. due to timeouts in the cache loader, this method requests the diff again using
295 : * the fallback algorithm {@link DiffAlgorithm#HISTOGRAM_NO_FALLBACK}.
296 : */
297 : private ImmutableMap<String, FileDiffOutput> getModifiedFilesForKeys(
298 : List<FileDiffCacheKey> keys, DiffOptions diffOptions) throws DiffNotAvailableException {
299 104 : ImmutableMap<FileDiffCacheKey, FileDiffOutput> fileDiffs = fileDiffCache.getAll(keys);
300 104 : List<FileDiffCacheKey> fallbackKeys = new ArrayList<>();
301 :
302 104 : ImmutableList.Builder<FileDiffOutput> result = ImmutableList.builder();
303 :
304 : // Use the fallback diff algorithm for negative results
305 104 : for (FileDiffCacheKey key : fileDiffs.keySet()) {
306 104 : FileDiffOutput diff = fileDiffs.get(key);
307 104 : if (diff.isNegative()) {
308 0 : FileDiffCacheKey fallbackKey =
309 0 : createFileDiffCacheKey(
310 0 : key.project(),
311 0 : key.oldCommit(),
312 0 : key.newCommit(),
313 0 : key.newFilePath(),
314 : // Use the fallback diff algorithm
315 : DiffAlgorithm.HISTOGRAM_NO_FALLBACK,
316 : // We don't enforce timeouts with the fallback algorithm. Timeouts were introduced
317 : // because of a bug in JGit that happens only when the histogram algorithm uses
318 : // Myers as fallback. See https://bugs.chromium.org/p/gerrit/issues/detail?id=487
319 : /* useTimeout= */ false,
320 0 : key.whitespace());
321 0 : fallbackKeys.add(fallbackKey);
322 0 : } else {
323 104 : result.add(diff);
324 : }
325 104 : }
326 104 : result.addAll(fileDiffCache.getAll(fallbackKeys).values());
327 104 : return mapByFilePath(result.build(), diffOptions);
328 : }
329 :
330 : /**
331 : * Map a collection of {@link FileDiffOutput} based on their file paths. The result map keys
332 : * represent the old file path for deleted files, or the new path otherwise.
333 : */
334 : private ImmutableMap<String, FileDiffOutput> mapByFilePath(
335 : ImmutableCollection<FileDiffOutput> fileDiffOutputs, DiffOptions diffOptions) {
336 104 : ImmutableMap.Builder<String, FileDiffOutput> diffs = ImmutableMap.builder();
337 :
338 104 : for (FileDiffOutput fileDiffOutput : fileDiffOutputs) {
339 104 : if (fileDiffOutput.isEmpty()
340 104 : || (diffOptions.skipFilesWithAllEditsDueToRebase() && allDueToRebase(fileDiffOutput))) {
341 2 : continue;
342 : }
343 104 : if (fileDiffOutput.changeType() == ChangeType.DELETED) {
344 23 : diffs.put(fileDiffOutput.oldPath().get(), fileDiffOutput);
345 : } else {
346 104 : diffs.put(fileDiffOutput.newPath().get(), fileDiffOutput);
347 : }
348 104 : }
349 104 : return diffs.build();
350 : }
351 :
352 : private static boolean allDueToRebase(FileDiffOutput fileDiffOutput) {
353 104 : return fileDiffOutput.allEditsDueToRebase()
354 2 : && !(fileDiffOutput.changeType() == ChangeType.RENAMED
355 104 : || fileDiffOutput.changeType() == ChangeType.COPIED);
356 : }
357 :
358 : private boolean isMergeAgainstParent(ComparisonType cmp, Project.NameKey project, ObjectId commit)
359 : throws IOException {
360 104 : return (cmp.isAgainstParent() && baseCommitUtil.getNumParents(project, commit) > 1);
361 : }
362 :
363 : private static ModifiedFilesCacheKey createModifiedFilesKey(
364 : Project.NameKey project, ObjectId aCommit, ObjectId bCommit) {
365 104 : return ModifiedFilesCacheKey.builder()
366 104 : .project(project)
367 104 : .aCommit(aCommit)
368 104 : .bCommit(bCommit)
369 104 : .renameScore(RENAME_SCORE)
370 104 : .build();
371 : }
372 :
373 : private static FileDiffCacheKey createFileDiffCacheKey(
374 : Project.NameKey project,
375 : ObjectId aCommit,
376 : ObjectId bCommit,
377 : String newPath,
378 : DiffAlgorithm diffAlgorithm,
379 : boolean useTimeout,
380 : @Nullable Whitespace whitespace) {
381 104 : whitespace = whitespace == null ? DEFAULT_WHITESPACE : whitespace;
382 104 : return FileDiffCacheKey.builder()
383 104 : .project(project)
384 104 : .oldCommit(aCommit)
385 104 : .newCommit(bCommit)
386 104 : .newFilePath(newPath)
387 104 : .renameScore(RENAME_SCORE)
388 104 : .diffAlgorithm(diffAlgorithm)
389 104 : .whitespace(whitespace)
390 104 : .useTimeout(useTimeout)
391 104 : .build();
392 : }
393 :
394 : /** Loads the modified file paths between two commits without inspecting the diff cache. */
395 : private static Map<String, ModifiedFile> loadModifiedFilesWithoutCache(
396 : Project.NameKey project, DiffParameters diffParams, RevWalk revWalk, Config repoConfig)
397 : throws DiffNotAvailableException {
398 3 : ObjectId newCommit = diffParams.newCommit();
399 3 : ObjectId oldCommit = diffParams.baseCommit();
400 : try {
401 3 : ObjectReader reader = revWalk.getObjectReader();
402 : List<DiffEntry> diffEntries;
403 3 : try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
404 3 : df.setReader(reader, repoConfig);
405 3 : df.setDetectRenames(false);
406 3 : diffEntries = df.scan(oldCommit.equals(ObjectId.zeroId()) ? null : oldCommit, newCommit);
407 : }
408 3 : List<ModifiedFile> modifiedFiles =
409 3 : diffEntries.stream()
410 3 : .map(
411 : entry ->
412 3 : ModifiedFile.builder()
413 3 : .changeType(toChangeType(entry.getChangeType()))
414 3 : .oldPath(getGitPath(entry.getOldPath()))
415 3 : .newPath(getGitPath(entry.getNewPath()))
416 3 : .build())
417 3 : .collect(Collectors.toList());
418 3 : return DiffUtil.mergeRewrittenModifiedFiles(modifiedFiles).stream()
419 3 : .collect(ImmutableMap.toImmutableMap(ModifiedFile::getDefaultPath, Function.identity()));
420 0 : } catch (IOException e) {
421 0 : throw new DiffNotAvailableException(
422 0 : String.format(
423 : "Failed to compute the modified files for project '%s',"
424 : + " old commit '%s', new commit '%s'.",
425 0 : project, oldCommit.name(), newCommit.name()),
426 : e);
427 : }
428 : }
429 :
430 : private static Optional<String> getGitPath(String path) {
431 3 : return path.equals(DiffEntry.DEV_NULL) ? Optional.empty() : Optional.of(path);
432 : }
433 :
434 : private static Patch.ChangeType toChangeType(DiffEntry.ChangeType changeType) {
435 3 : if (!changeTypeMap.containsKey(changeType)) {
436 0 : throw new IllegalArgumentException("Unsupported type " + changeType);
437 : }
438 3 : return changeTypeMap.get(changeType);
439 : }
440 :
441 : @AutoValue
442 104 : abstract static class DiffParameters {
443 : abstract Project.NameKey project();
444 :
445 : abstract ObjectId newCommit();
446 :
447 : /**
448 : * Base commit represents the old commit of the diff. For diffs against the root commit, this
449 : * should be set to {@link ObjectId#zeroId()}.
450 : */
451 : abstract ObjectId baseCommit();
452 :
453 : abstract ComparisonType comparisonType();
454 :
455 : @Nullable
456 : abstract Integer parent();
457 :
458 : /** Compute the diff for {@value Patch#COMMIT_MSG} and {@link Patch#MERGE_LIST} only. */
459 : @Nullable
460 : abstract Boolean skipFiles();
461 :
462 : static Builder builder() {
463 104 : return new AutoValue_DiffOperationsImpl_DiffParameters.Builder();
464 : }
465 :
466 : @AutoValue.Builder
467 104 : abstract static class Builder {
468 :
469 : abstract Builder project(Project.NameKey project);
470 :
471 : abstract Builder newCommit(ObjectId newCommit);
472 :
473 : abstract Builder baseCommit(ObjectId baseCommit);
474 :
475 : abstract Builder parent(@Nullable Integer parent);
476 :
477 : abstract Builder skipFiles(@Nullable Boolean skipFiles);
478 :
479 : abstract Builder comparisonType(ComparisonType comparisonType);
480 :
481 : public abstract DiffParameters build();
482 : }
483 : }
484 :
485 : /** Compute Diff parameters - the base commit and the comparison type - using the input args. */
486 : private DiffParameters computeDiffParameters(
487 : Project.NameKey project, ObjectId newCommit, Integer parent) throws IOException {
488 : DiffParameters.Builder result =
489 104 : DiffParameters.builder().project(project).newCommit(newCommit).parent(parent);
490 104 : if (parent > 0) {
491 19 : result.baseCommit(baseCommitUtil.getBaseCommit(project, newCommit, parent));
492 19 : result.comparisonType(ComparisonType.againstParent(parent));
493 19 : return result.build();
494 : }
495 104 : int numParents = baseCommitUtil.getNumParents(project, newCommit);
496 104 : if (numParents == 0) {
497 32 : result.baseCommit(ObjectId.zeroId());
498 32 : result.comparisonType(ComparisonType.againstRoot());
499 32 : return result.build();
500 : }
501 100 : if (numParents == 1) {
502 100 : result.baseCommit(baseCommitUtil.getBaseCommit(project, newCommit, parent));
503 100 : result.comparisonType(ComparisonType.againstParent(1));
504 100 : return result.build();
505 : }
506 31 : if (numParents > 2) {
507 4 : logger.atFine().log(
508 : "Diff against auto-merge for merge commits "
509 : + "with more than two parents is not supported. Commit %s has %d parents."
510 : + " Falling back to the diff against the first parent.",
511 : newCommit, numParents);
512 4 : result.baseCommit(baseCommitUtil.getBaseCommit(project, newCommit, 1).getId());
513 4 : result.comparisonType(ComparisonType.againstParent(1));
514 4 : result.skipFiles(true);
515 : } else {
516 31 : result.baseCommit(baseCommitUtil.getBaseCommit(project, newCommit, null));
517 31 : result.comparisonType(ComparisonType.againstAutoMerge());
518 : }
519 31 : return result.build();
520 : }
521 : }
|
