Line data Source code
1 : // Copyright (C) 2021 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.patch;
16 :
17 : import static com.google.gerrit.server.project.ProjectCache.illegalState;
18 :
19 : import com.google.gerrit.common.Nullable;
20 : import com.google.gerrit.common.data.PatchScript;
21 : import com.google.gerrit.entities.LabelId;
22 : import com.google.gerrit.entities.LabelType;
23 : import com.google.gerrit.entities.Patch;
24 : import com.google.gerrit.entities.Patch.ChangeType;
25 : import com.google.gerrit.entities.PatchSet;
26 : import com.google.gerrit.entities.PatchSetApproval;
27 : import com.google.gerrit.entities.Project;
28 : import com.google.gerrit.exceptions.StorageException;
29 : import com.google.gerrit.extensions.client.DiffPreferencesInfo;
30 : import com.google.gerrit.extensions.client.DiffPreferencesInfo.Whitespace;
31 : import com.google.gerrit.extensions.restapi.AuthException;
32 : import com.google.gerrit.server.CurrentUser;
33 : import com.google.gerrit.server.config.GerritServerConfig;
34 : import com.google.gerrit.server.diff.DiffInfoCreator;
35 : import com.google.gerrit.server.git.GitRepositoryManager;
36 : import com.google.gerrit.server.git.LargeObjectException;
37 : import com.google.gerrit.server.git.validators.CommentCumulativeSizeValidator;
38 : import com.google.gerrit.server.notedb.ChangeNotes;
39 : import com.google.gerrit.server.patch.filediff.FileDiffOutput;
40 : import com.google.gerrit.server.permissions.PermissionBackendException;
41 : import com.google.gerrit.server.project.InvalidChangeOperationException;
42 : import com.google.gerrit.server.project.ProjectCache;
43 : import com.google.gerrit.server.project.ProjectState;
44 : import com.google.inject.Inject;
45 : import java.io.IOException;
46 : import java.util.ArrayList;
47 : import java.util.Arrays;
48 : import java.util.List;
49 : import java.util.Map;
50 : import java.util.Optional;
51 : import java.util.stream.Collectors;
52 : import org.eclipse.jgit.diff.DiffFormatter;
53 : import org.eclipse.jgit.internal.JGitText;
54 : import org.eclipse.jgit.lib.Config;
55 : import org.eclipse.jgit.lib.Repository;
56 : import org.eclipse.jgit.util.RawParseUtils;
57 : import org.eclipse.jgit.util.TemporaryBuffer;
58 :
59 : /**
60 : * This class is used on submit to compute the diff between the latest approved patch-set, and the
61 : * current submitted patch-set.
62 : *
63 : * <p>Latest approved patch-set is defined by the latest patch-set which has Code-Review label voted
64 : * with the maximum possible value.
65 : *
66 : * <p>If the latest approved patch-set is the same as the submitted patch-set, the diff will be
67 : * empty.
68 : *
69 : * <p>We exclude the magic files from the returned diff to make it shorter and more concise.
70 : */
71 : public class SubmitWithStickyApprovalDiff {
72 : private static final int HEAP_EST_SIZE = 32 * 1024;
73 : private static final int DEFAULT_POST_SUBMIT_SIZE_LIMIT = 300 * 1024; // 300 KiB
74 :
75 : private final DiffOperations diffOperations;
76 : private final ProjectCache projectCache;
77 : private final PatchScriptFactory.Factory patchScriptFactoryFactory;
78 : private final GitRepositoryManager repositoryManager;
79 : private final int maxCumulativeSize;
80 :
81 : @Inject
82 : SubmitWithStickyApprovalDiff(
83 : DiffOperations diffOperations,
84 : ProjectCache projectCache,
85 : PatchScriptFactory.Factory patchScriptFactoryFactory,
86 : GitRepositoryManager repositoryManager,
87 53 : @GerritServerConfig Config serverConfig) {
88 53 : this.diffOperations = diffOperations;
89 53 : this.projectCache = projectCache;
90 53 : this.patchScriptFactoryFactory = patchScriptFactoryFactory;
91 53 : this.repositoryManager = repositoryManager;
92 : // (November 2021) We define the max cumulative comment size to 300 KIB since it's a reasonable
93 : // size that is large enough for all purposes but not too large to choke the change index by
94 : // exceeding the cumulative comment size limit (new comments are not allowed once the limit
95 : // is reached). At Google, the change index limit is 5MB, while the cumulative size limit is
96 : // set at 3MB. In this example, we can reach at most 3.3MB hence we ensure not to exceed the
97 : // limit of 5MB.
98 : // The reason we exclude the post submit diff from the cumulative comment size limit is
99 : // just because change messages not currently being validated. Change messages are still
100 : // counted towards the limit, though.
101 53 : maxCumulativeSize =
102 53 : serverConfig.getInt(
103 : "change",
104 : "cumulativeCommentSizeLimit",
105 : CommentCumulativeSizeValidator.DEFAULT_CUMULATIVE_COMMENT_SIZE_LIMIT);
106 53 : }
107 :
108 : public String apply(ChangeNotes notes, CurrentUser currentUser)
109 : throws AuthException, IOException, PermissionBackendException,
110 : InvalidChangeOperationException {
111 53 : PatchSet currentPatchset = notes.getCurrentPatchSet();
112 :
113 53 : PatchSet.Id latestApprovedPatchsetId = getLatestApprovedPatchsetId(notes);
114 53 : if (latestApprovedPatchsetId.get() == currentPatchset.id().get()) {
115 : // If the latest approved patchset is the current patchset, no need to return anything.
116 53 : return "";
117 : }
118 5 : StringBuilder diff =
119 : new StringBuilder(
120 5 : String.format(
121 5 : "\n\n%d is the latest approved patch-set.\n", latestApprovedPatchsetId.get()));
122 5 : Map<String, FileDiffOutput> modifiedFiles =
123 5 : listModifiedFiles(
124 5 : notes.getProjectName(),
125 : currentPatchset,
126 5 : notes.getPatchSets().get(latestApprovedPatchsetId));
127 :
128 : // To make the message a bit more concise, we skip the magic files.
129 5 : List<FileDiffOutput> modifiedFilesList =
130 5 : modifiedFiles.values().stream()
131 5 : .filter(p -> !Patch.isMagic(p.newPath().orElse("")))
132 5 : .collect(Collectors.toList());
133 :
134 5 : if (modifiedFilesList.isEmpty()) {
135 2 : diff.append(
136 : "No files were changed between the latest approved patch-set and the submitted one.\n");
137 2 : return diff.toString();
138 : }
139 :
140 4 : diff.append("The change was submitted with unreviewed changes in the following files:\n\n");
141 4 : TemporaryBuffer.Heap buffer =
142 : new TemporaryBuffer.Heap(
143 4 : Math.min(HEAP_EST_SIZE, DEFAULT_POST_SUBMIT_SIZE_LIMIT),
144 : DEFAULT_POST_SUBMIT_SIZE_LIMIT);
145 4 : try (Repository repository = repositoryManager.openRepository(notes.getProjectName());
146 4 : DiffFormatter formatter = new DiffFormatter(buffer)) {
147 4 : formatter.setRepository(repository);
148 4 : formatter.setDetectRenames(true);
149 4 : boolean isDiffTooLarge = false;
150 4 : List<String> formatterResult = null;
151 : try {
152 4 : formatter.format(
153 4 : modifiedFilesList.get(0).oldCommitId(), modifiedFilesList.get(0).newCommitId());
154 : // This returns the diff for all the files.
155 4 : formatterResult =
156 4 : Arrays.stream(RawParseUtils.decode(buffer.toByteArray()).split("\n"))
157 4 : .collect(Collectors.toList());
158 1 : } catch (IOException e) {
159 1 : if (JGitText.get().inMemoryBufferLimitExceeded.equals(e.getMessage())) {
160 1 : isDiffTooLarge = true;
161 : } else {
162 0 : throw e;
163 : }
164 4 : }
165 4 : if (formatterResult != null) {
166 4 : int addedBytes = formatterResult.stream().mapToInt(String::length).sum();
167 4 : if (!CommentCumulativeSizeValidator.isEnoughSpace(notes, addedBytes, maxCumulativeSize)) {
168 1 : isDiffTooLarge = true;
169 : }
170 : }
171 4 : for (FileDiffOutput fileDiff : modifiedFilesList) {
172 4 : diff.append(
173 4 : getDiffForFile(
174 : notes,
175 4 : currentPatchset.id(),
176 : latestApprovedPatchsetId,
177 : fileDiff,
178 : currentUser,
179 : formatterResult,
180 : isDiffTooLarge));
181 4 : }
182 : }
183 4 : return diff.toString();
184 : }
185 :
186 : private String getDiffForFile(
187 : ChangeNotes notes,
188 : PatchSet.Id currentPatchsetId,
189 : PatchSet.Id latestApprovedPatchsetId,
190 : FileDiffOutput fileDiffOutput,
191 : CurrentUser currentUser,
192 : @Nullable List<String> formatterResult,
193 : boolean isDiffTooLarge)
194 : throws AuthException, InvalidChangeOperationException, IOException,
195 : PermissionBackendException {
196 4 : StringBuilder diff =
197 : new StringBuilder(
198 4 : String.format(
199 : "```\nThe name of the file: %s\nInsertions: %d, Deletions: %d.\n\n",
200 4 : fileDiffOutput.newPath().isPresent()
201 4 : ? fileDiffOutput.newPath().get()
202 4 : : fileDiffOutput.oldPath().get(),
203 4 : fileDiffOutput.insertions(),
204 4 : fileDiffOutput.deletions()));
205 4 : DiffPreferencesInfo diffPreferencesInfo = createDefaultDiffPreferencesInfo();
206 4 : PatchScriptFactory patchScriptFactory =
207 4 : patchScriptFactoryFactory.create(
208 : notes,
209 4 : fileDiffOutput.newPath().isPresent()
210 4 : ? fileDiffOutput.newPath().get()
211 4 : : fileDiffOutput.oldPath().get(),
212 : latestApprovedPatchsetId,
213 : currentPatchsetId,
214 : diffPreferencesInfo,
215 : currentUser);
216 4 : PatchScript patchScript = null;
217 : try {
218 : // TODO(paiking): we can get rid of this call to optimize by checking the diff for renames.
219 4 : patchScript = patchScriptFactory.call();
220 0 : } catch (LargeObjectException exception) {
221 0 : diff.append("The file content is too large for showing the full diff. \n\n");
222 0 : return diff.toString();
223 4 : }
224 4 : if (patchScript.getChangeType() == ChangeType.RENAMED) {
225 1 : diff.append(
226 1 : String.format(
227 : "The file %s was renamed to %s\n",
228 1 : fileDiffOutput.oldPath().get(), fileDiffOutput.newPath().get()));
229 : }
230 4 : if (isDiffTooLarge) {
231 1 : diff.append("The diff is too large to show. Please review the diff.");
232 1 : diff.append("\n```\n");
233 1 : return diff.toString();
234 : }
235 : // This filters only the file we need.
236 : // TODO(paiking): we can make this more efficient by mapping the files to their respective
237 : // diffs prior to this method, such that we need to go over the diff only once.
238 4 : diff.append(getDiffForFile(patchScript, formatterResult));
239 : // This line (and the ``` above) are useful for formatting in the web UI.
240 4 : diff.append("\n```\n");
241 4 : return diff.toString();
242 : }
243 :
244 : /**
245 : * Show patch set as unified difference for a specific file. We on purpose are not using {@link
246 : * DiffInfoCreator} since we'd like to get the original git/JGit style diff.
247 : */
248 : public String getDiffForFile(PatchScript patchScript, List<String> formatterResult) {
249 : // only return information about the current file, and not about files that are not
250 : // relevant. DiffFormatter returns other potential files because of rebases, which we can
251 : // ignore.
252 4 : List<String> modifiedFormatterResult = new ArrayList<>();
253 4 : int indexOfFormatterResult = 0;
254 4 : while (formatterResult.size() > indexOfFormatterResult
255 : && !formatterResult
256 4 : .get(indexOfFormatterResult)
257 4 : .equals(
258 4 : String.format(
259 : "diff --git a/%s b/%s",
260 4 : patchScript.getOldName() != null
261 1 : ? patchScript.getOldName()
262 4 : : patchScript.getNewName(),
263 4 : patchScript.getNewName()))) {
264 1 : indexOfFormatterResult++;
265 : }
266 : // remove non user friendly information.
267 4 : while (formatterResult.size() > indexOfFormatterResult
268 4 : && !formatterResult.get(indexOfFormatterResult).startsWith("@@")) {
269 4 : indexOfFormatterResult++;
270 : }
271 4 : for (; indexOfFormatterResult < formatterResult.size(); indexOfFormatterResult++) {
272 4 : if (formatterResult.get(indexOfFormatterResult).startsWith("diff --git")) {
273 1 : break;
274 : }
275 4 : modifiedFormatterResult.add(formatterResult.get(indexOfFormatterResult));
276 : }
277 4 : if (modifiedFormatterResult.size() == 0) {
278 : // This happens for diffs that are just renames, but we already account for renames.
279 1 : return "";
280 : }
281 4 : return modifiedFormatterResult.stream()
282 4 : .filter(s -> !s.equals("\\ No newline at end of file"))
283 4 : .collect(Collectors.joining("\n"));
284 : }
285 :
286 : private DiffPreferencesInfo createDefaultDiffPreferencesInfo() {
287 4 : DiffPreferencesInfo diffPreferencesInfo = new DiffPreferencesInfo();
288 4 : diffPreferencesInfo.ignoreWhitespace = Whitespace.IGNORE_NONE;
289 4 : diffPreferencesInfo.intralineDifference = true;
290 4 : return diffPreferencesInfo;
291 : }
292 :
293 : private PatchSet.Id getLatestApprovedPatchsetId(ChangeNotes notes) {
294 53 : ProjectState projectState =
295 53 : projectCache.get(notes.getProjectName()).orElseThrow(illegalState(notes.getProjectName()));
296 53 : PatchSet.Id maxPatchSetId = PatchSet.id(notes.getChangeId(), 1);
297 53 : for (PatchSetApproval patchSetApproval : notes.getApprovals().onlyNonCopied().values()) {
298 46 : if (!patchSetApproval.label().equals(LabelId.CODE_REVIEW)) {
299 10 : continue;
300 : }
301 46 : Optional<LabelType> lt =
302 46 : projectState.getLabelTypes(notes).byLabel(patchSetApproval.labelId());
303 46 : if (!lt.isPresent() || !lt.get().isMaxPositive(patchSetApproval)) {
304 4 : continue;
305 : }
306 46 : if (patchSetApproval.patchSetId().get() > maxPatchSetId.get()) {
307 10 : maxPatchSetId = patchSetApproval.patchSetId();
308 : }
309 46 : }
310 53 : return maxPatchSetId;
311 : }
312 :
313 : /**
314 : * Gets the list of modified files between the two latest patch-sets. Can be used to compute
315 : * difference in files between those two patch-sets.
316 : */
317 : private Map<String, FileDiffOutput> listModifiedFiles(
318 : Project.NameKey project, PatchSet ps, PatchSet priorPatchSet) {
319 : try {
320 5 : return diffOperations.listModifiedFiles(
321 5 : project, priorPatchSet.commitId(), ps.commitId(), DiffOptions.DEFAULTS);
322 0 : } catch (DiffNotAvailableException ex) {
323 0 : throw new StorageException(
324 : "failed to compute difference in files, so won't post diff messsage on submit although "
325 : + "the latest approved patch-set was not the same as the submitted patch-set.",
326 : ex);
327 : }
328 : }
329 : }
|
