Line data Source code
1 : // Copyright (C) 2016 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.server.mail.receive;
16 :
17 : import static com.google.gerrit.entities.Patch.PATCHSET_LEVEL;
18 : import static java.util.stream.Collectors.toList;
19 :
20 : import com.google.common.base.Strings;
21 : import com.google.common.collect.ImmutableList;
22 : import com.google.common.collect.ImmutableMap;
23 : import com.google.common.collect.Iterables;
24 : import com.google.common.flogger.FluentLogger;
25 : import com.google.gerrit.entities.Account;
26 : import com.google.gerrit.entities.Change;
27 : import com.google.gerrit.entities.HumanComment;
28 : import com.google.gerrit.entities.PatchSet;
29 : import com.google.gerrit.entities.Project;
30 : import com.google.gerrit.exceptions.StorageException;
31 : import com.google.gerrit.extensions.client.Side;
32 : import com.google.gerrit.extensions.registration.DynamicItem;
33 : import com.google.gerrit.extensions.registration.DynamicMap;
34 : import com.google.gerrit.extensions.registration.Extension;
35 : import com.google.gerrit.extensions.restapi.RestApiException;
36 : import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
37 : import com.google.gerrit.extensions.validators.CommentForValidation;
38 : import com.google.gerrit.extensions.validators.CommentValidationContext;
39 : import com.google.gerrit.extensions.validators.CommentValidationFailure;
40 : import com.google.gerrit.extensions.validators.CommentValidator;
41 : import com.google.gerrit.mail.HtmlParser;
42 : import com.google.gerrit.mail.MailComment;
43 : import com.google.gerrit.mail.MailHeaderParser;
44 : import com.google.gerrit.mail.MailMessage;
45 : import com.google.gerrit.mail.MailMetadata;
46 : import com.google.gerrit.mail.TextParser;
47 : import com.google.gerrit.server.ChangeMessagesUtil;
48 : import com.google.gerrit.server.CommentsUtil;
49 : import com.google.gerrit.server.PatchSetUtil;
50 : import com.google.gerrit.server.PublishCommentUtil;
51 : import com.google.gerrit.server.account.AccountCache;
52 : import com.google.gerrit.server.account.AccountState;
53 : import com.google.gerrit.server.account.Emails;
54 : import com.google.gerrit.server.approval.ApprovalsUtil;
55 : import com.google.gerrit.server.change.EmailReviewComments;
56 : import com.google.gerrit.server.config.UrlFormatter;
57 : import com.google.gerrit.server.extensions.events.CommentAdded;
58 : import com.google.gerrit.server.mail.MailFilter;
59 : import com.google.gerrit.server.mail.send.InboundEmailRejectionSender;
60 : import com.google.gerrit.server.mail.send.InboundEmailRejectionSender.InboundEmailError;
61 : import com.google.gerrit.server.mail.send.MessageIdGenerator;
62 : import com.google.gerrit.server.notedb.ChangeNotes;
63 : import com.google.gerrit.server.plugincontext.PluginSetContext;
64 : import com.google.gerrit.server.query.change.ChangeData;
65 : import com.google.gerrit.server.query.change.InternalChangeQuery;
66 : import com.google.gerrit.server.update.BatchUpdate;
67 : import com.google.gerrit.server.update.BatchUpdateOp;
68 : import com.google.gerrit.server.update.ChangeContext;
69 : import com.google.gerrit.server.update.PostUpdateContext;
70 : import com.google.gerrit.server.update.RetryHelper;
71 : import com.google.gerrit.server.update.UpdateException;
72 : import com.google.gerrit.server.util.ManualRequestContext;
73 : import com.google.gerrit.server.util.OneOffRequestContext;
74 : import com.google.gerrit.server.util.time.TimeUtil;
75 : import com.google.inject.Inject;
76 : import com.google.inject.Provider;
77 : import com.google.inject.Singleton;
78 : import java.io.IOException;
79 : import java.util.ArrayList;
80 : import java.util.Collection;
81 : import java.util.HashMap;
82 : import java.util.HashSet;
83 : import java.util.List;
84 : import java.util.Map;
85 : import java.util.Optional;
86 : import java.util.Set;
87 :
88 : /** A service that can attach the comments from a {@link MailMessage} to a change. */
89 : @Singleton
90 : public class MailProcessor {
91 4 : private static final FluentLogger logger = FluentLogger.forEnclosingClass();
92 :
93 : private static final ImmutableMap<MailComment.CommentType, CommentForValidation.CommentType>
94 4 : MAIL_COMMENT_TYPE_TO_VALIDATION_TYPE =
95 4 : ImmutableMap.of(
96 : MailComment.CommentType.PATCHSET_LEVEL,
97 : CommentForValidation.CommentType.CHANGE_MESSAGE,
98 : MailComment.CommentType.FILE_COMMENT, CommentForValidation.CommentType.FILE_COMMENT,
99 : MailComment.CommentType.INLINE_COMMENT,
100 : CommentForValidation.CommentType.INLINE_COMMENT);
101 :
102 : private final Emails emails;
103 : private final InboundEmailRejectionSender.Factory emailRejectionSender;
104 : private final RetryHelper retryHelper;
105 : private final ChangeMessagesUtil changeMessagesUtil;
106 : private final CommentsUtil commentsUtil;
107 : private final OneOffRequestContext oneOffRequestContext;
108 : private final PatchSetUtil psUtil;
109 : private final Provider<InternalChangeQuery> queryProvider;
110 : private final DynamicMap<MailFilter> mailFilters;
111 : private final EmailReviewComments.Factory outgoingMailFactory;
112 : private final CommentAdded commentAdded;
113 : private final ApprovalsUtil approvalsUtil;
114 : private final AccountCache accountCache;
115 : private final DynamicItem<UrlFormatter> urlFormatter;
116 : private final PluginSetContext<CommentValidator> commentValidators;
117 : private final MessageIdGenerator messageIdGenerator;
118 :
119 : @Inject
120 : public MailProcessor(
121 : Emails emails,
122 : InboundEmailRejectionSender.Factory emailRejectionSender,
123 : RetryHelper retryHelper,
124 : ChangeMessagesUtil changeMessagesUtil,
125 : CommentsUtil commentsUtil,
126 : OneOffRequestContext oneOffRequestContext,
127 : PatchSetUtil psUtil,
128 : Provider<InternalChangeQuery> queryProvider,
129 : DynamicMap<MailFilter> mailFilters,
130 : EmailReviewComments.Factory outgoingMailFactory,
131 : ApprovalsUtil approvalsUtil,
132 : CommentAdded commentAdded,
133 : AccountCache accountCache,
134 : DynamicItem<UrlFormatter> urlFormatter,
135 : PluginSetContext<CommentValidator> commentValidators,
136 4 : MessageIdGenerator messageIdGenerator) {
137 4 : this.emails = emails;
138 4 : this.emailRejectionSender = emailRejectionSender;
139 4 : this.retryHelper = retryHelper;
140 4 : this.changeMessagesUtil = changeMessagesUtil;
141 4 : this.commentsUtil = commentsUtil;
142 4 : this.oneOffRequestContext = oneOffRequestContext;
143 4 : this.psUtil = psUtil;
144 4 : this.queryProvider = queryProvider;
145 4 : this.mailFilters = mailFilters;
146 4 : this.outgoingMailFactory = outgoingMailFactory;
147 4 : this.commentAdded = commentAdded;
148 4 : this.approvalsUtil = approvalsUtil;
149 4 : this.accountCache = accountCache;
150 4 : this.urlFormatter = urlFormatter;
151 4 : this.commentValidators = commentValidators;
152 4 : this.messageIdGenerator = messageIdGenerator;
153 4 : }
154 :
155 : /**
156 : * Parses comments from a {@link MailMessage} and persists them on the change.
157 : *
158 : * @param message {@link MailMessage} to process
159 : */
160 : public void process(MailMessage message) throws RestApiException, UpdateException {
161 3 : retryHelper
162 3 : .changeUpdate(
163 : "processCommentsReceivedByEmail",
164 : buf -> {
165 3 : processImpl(buf, message);
166 3 : return null;
167 : })
168 3 : .call();
169 3 : }
170 :
171 : private void processImpl(BatchUpdate.Factory buf, MailMessage message)
172 : throws UpdateException, RestApiException, IOException {
173 3 : for (Extension<MailFilter> filter : mailFilters) {
174 3 : if (!filter.getProvider().get().shouldProcessMessage(message)) {
175 1 : logger.atWarning().log(
176 : "Message %s filtered by plugin %s %s. Will delete message.",
177 1 : message.id(), filter.getPluginName(), filter.getExportName());
178 1 : return;
179 : }
180 3 : }
181 :
182 3 : MailMetadata metadata = MailHeaderParser.parse(message);
183 :
184 3 : if (!metadata.hasRequiredFields()) {
185 2 : logger.atSevere().log(
186 : "Message %s is missing required metadata, have %s. Will delete message.",
187 2 : message.id(), metadata);
188 2 : sendRejectionEmail(message, InboundEmailError.PARSING_ERROR);
189 2 : return;
190 : }
191 :
192 2 : Set<Account.Id> accountIds = emails.getAccountFor(metadata.author);
193 :
194 2 : if (accountIds.size() != 1) {
195 0 : logger.atSevere().log(
196 : "Address %s could not be matched to a unique account. It was matched to %s."
197 : + " Will delete message.",
198 : metadata.author, accountIds);
199 :
200 : // We don't want to send an email if no accounts are linked to it.
201 0 : if (accountIds.size() > 1) {
202 0 : sendRejectionEmail(message, InboundEmailError.UNKNOWN_ACCOUNT);
203 : }
204 0 : return;
205 : }
206 2 : Account.Id accountId = accountIds.iterator().next();
207 2 : Optional<AccountState> accountState = accountCache.get(accountId);
208 2 : if (!accountState.isPresent()) {
209 0 : logger.atWarning().log("Mail: Account %s doesn't exist. Will delete message.", accountId);
210 0 : return;
211 : }
212 2 : if (!accountState.get().account().isActive()) {
213 1 : logger.atWarning().log("Mail: Account %s is inactive. Will delete message.", accountId);
214 1 : sendRejectionEmail(message, InboundEmailError.INACTIVE_ACCOUNT);
215 1 : return;
216 : }
217 :
218 2 : persistComments(buf, message, metadata, accountId);
219 2 : }
220 :
221 : private void sendRejectionEmail(MailMessage message, InboundEmailError reason) {
222 : try {
223 2 : InboundEmailRejectionSender emailSender =
224 2 : emailRejectionSender.create(message.from(), message.id(), reason);
225 2 : emailSender.setMessageId(messageIdGenerator.fromMailMessage(message));
226 2 : emailSender.send();
227 0 : } catch (Exception e) {
228 0 : logger.atSevere().withCause(e).log("Cannot send email to warn for an error");
229 2 : }
230 2 : }
231 :
232 : private void persistComments(
233 : BatchUpdate.Factory buf, MailMessage message, MailMetadata metadata, Account.Id sender)
234 : throws UpdateException, RestApiException {
235 2 : try (ManualRequestContext ctx = oneOffRequestContext.openAs(sender)) {
236 2 : List<ChangeData> changeDataList =
237 : queryProvider
238 2 : .get()
239 2 : .enforceVisibility(true)
240 2 : .byLegacyChangeId(Change.id(metadata.changeNumber));
241 2 : if (changeDataList.isEmpty()) {
242 1 : sendRejectionEmail(message, InboundEmailError.CHANGE_NOT_FOUND);
243 1 : return;
244 : }
245 2 : if (changeDataList.size() != 1) {
246 0 : logger.atSevere().log(
247 : "Message %s references unique change %s,"
248 : + " but there are %d matching changes in the index."
249 : + " Will delete message.",
250 0 : message.id(), metadata.changeNumber, changeDataList.size());
251 :
252 0 : sendRejectionEmail(message, InboundEmailError.INTERNAL_EXCEPTION);
253 0 : return;
254 : }
255 2 : ChangeData cd = Iterables.getOnlyElement(changeDataList);
256 2 : if (existingMessageIds(cd).contains(message.id())) {
257 1 : logger.atInfo().log("Message %s was already processed. Will delete message.", message.id());
258 1 : return;
259 : }
260 : // Get all comments; filter and sort them to get the original list of
261 : // comments from the outbound email.
262 : // TODO(hiesel) Also filter by original comment author.
263 2 : Collection<HumanComment> comments =
264 2 : cd.publishedComments().stream()
265 2 : .filter(c -> (c.writtenOn.getTime() / 1000) == (metadata.timestamp.getTime() / 1000))
266 2 : .sorted(CommentsUtil.COMMENT_ORDER)
267 2 : .collect(toList());
268 2 : Project.NameKey project = cd.project();
269 :
270 : // If URL is not defined, we won't be able to parse line comments. We still attempt to get the
271 : // other ones.
272 2 : String changeUrl =
273 : urlFormatter
274 2 : .get()
275 2 : .getChangeViewUrl(cd.project(), cd.getId())
276 2 : .orElse("http://gerrit.invalid/");
277 :
278 : List<MailComment> parsedComments;
279 2 : if (useHtmlParser(message)) {
280 0 : parsedComments = HtmlParser.parse(message, comments, changeUrl);
281 : } else {
282 2 : parsedComments = TextParser.parse(message, comments, changeUrl);
283 : }
284 :
285 2 : if (parsedComments.isEmpty()) {
286 0 : logger.atWarning().log(
287 0 : "Could not parse any comments from %s. Will delete message.", message.id());
288 0 : sendRejectionEmail(message, InboundEmailError.PARSING_ERROR);
289 0 : return;
290 : }
291 :
292 2 : ImmutableList<CommentForValidation> parsedCommentsForValidation =
293 2 : parsedComments.stream()
294 2 : .map(
295 : comment ->
296 2 : CommentForValidation.create(
297 : CommentForValidation.CommentSource.HUMAN,
298 2 : MAIL_COMMENT_TYPE_TO_VALIDATION_TYPE.get(comment.getType()),
299 2 : comment.getMessage(),
300 2 : comment.getMessage().length()))
301 2 : .collect(ImmutableList.toImmutableList());
302 2 : CommentValidationContext commentValidationCtx =
303 2 : CommentValidationContext.create(
304 2 : cd.change().getChangeId(),
305 2 : cd.change().getProject().get(),
306 2 : cd.change().getDest().branch());
307 2 : ImmutableList<CommentValidationFailure> commentValidationFailures =
308 2 : PublishCommentUtil.findInvalidComments(
309 : commentValidationCtx, commentValidators, parsedCommentsForValidation);
310 2 : if (!commentValidationFailures.isEmpty()) {
311 1 : sendRejectionEmail(message, InboundEmailError.COMMENT_REJECTED);
312 1 : return;
313 : }
314 :
315 2 : Op o = new Op(PatchSet.id(cd.getId(), metadata.patchSet), parsedComments, message.id());
316 2 : BatchUpdate batchUpdate = buf.create(project, ctx.getUser(), TimeUtil.now());
317 2 : batchUpdate.addOp(cd.getId(), o);
318 2 : batchUpdate.execute();
319 1 : }
320 2 : }
321 :
322 : private class Op implements BatchUpdateOp {
323 : private final PatchSet.Id psId;
324 : private final List<MailComment> parsedComments;
325 : private final String tag;
326 : private String mailMessage;
327 : private List<HumanComment> comments;
328 : private PatchSet patchSet;
329 : private ChangeNotes notes;
330 :
331 2 : private Op(PatchSet.Id psId, List<MailComment> parsedComments, String messageId) {
332 2 : this.psId = psId;
333 2 : this.parsedComments = parsedComments;
334 2 : this.tag = "mailMessageId=" + messageId;
335 2 : }
336 :
337 : @Override
338 : public boolean updateChange(ChangeContext ctx) throws UnprocessableEntityException {
339 2 : patchSet = psUtil.get(ctx.getNotes(), psId);
340 2 : notes = ctx.getNotes();
341 2 : if (patchSet == null) {
342 0 : throw new StorageException("patch set not found: " + psId);
343 : }
344 :
345 2 : mailMessage =
346 2 : changeMessagesUtil.setChangeMessage(ctx.getUpdate(psId), generateChangeMessage(), tag);
347 2 : comments = new ArrayList<>();
348 2 : for (MailComment c : parsedComments) {
349 2 : comments.add(
350 2 : persistentCommentFromMailComment(ctx, c, targetPatchSetForComment(ctx, c, patchSet)));
351 2 : }
352 2 : commentsUtil.putHumanComments(
353 2 : ctx.getUpdate(ctx.getChange().currentPatchSetId()),
354 : HumanComment.Status.PUBLISHED,
355 : comments);
356 :
357 2 : return true;
358 : }
359 :
360 : @Override
361 : public void postUpdate(PostUpdateContext ctx) throws Exception {
362 2 : String patchSetComment = null;
363 2 : if (parsedComments.get(0).getType() == MailComment.CommentType.PATCHSET_LEVEL) {
364 2 : patchSetComment = parsedComments.get(0).getMessage();
365 : }
366 : // Send email notifications
367 2 : outgoingMailFactory
368 2 : .create(
369 : ctx,
370 : patchSet,
371 2 : notes.getMetaId(),
372 : mailMessage,
373 : comments,
374 : patchSetComment,
375 2 : ImmutableList.of())
376 2 : .sendAsync();
377 : // Get previous approvals from this user
378 2 : Map<String, Short> approvals = new HashMap<>();
379 2 : approvalsUtil
380 2 : .byPatchSetUser(notes, psId, ctx.getAccountId())
381 2 : .forEach(a -> approvals.put(a.label(), a.value()));
382 : // Fire Gerrit event. Note that approvals can't be granted via email, so old and new approvals
383 : // are always the same here.
384 2 : commentAdded.fire(
385 2 : ctx.getChangeData(notes),
386 : patchSet,
387 2 : ctx.getAccount(),
388 : mailMessage,
389 : approvals,
390 : approvals,
391 2 : ctx.getWhen());
392 2 : }
393 :
394 : private String generateChangeMessage() {
395 2 : String changeMsg = "Patch Set " + psId.get() + ":";
396 2 : changeMsg += "\n\n" + numComments(parsedComments.size());
397 2 : return changeMsg;
398 : }
399 :
400 : private PatchSet targetPatchSetForComment(
401 : ChangeContext ctx, MailComment mailComment, PatchSet current) {
402 2 : if (mailComment.getInReplyTo() != null) {
403 1 : return psUtil.get(
404 1 : ctx.getNotes(),
405 1 : PatchSet.id(ctx.getChange().getId(), mailComment.getInReplyTo().key.patchSetId));
406 : }
407 2 : return current;
408 : }
409 :
410 : private HumanComment persistentCommentFromMailComment(
411 : ChangeContext ctx, MailComment mailComment, PatchSet patchSetForComment) {
412 : String fileName;
413 : // The patch set that this comment is based on is different if this
414 : // comment was sent in reply to a comment on a previous patch set.
415 : Side side;
416 2 : if (mailComment.getType() == MailComment.CommentType.PATCHSET_LEVEL) {
417 2 : fileName = PATCHSET_LEVEL;
418 : // Patchset comments do not have side.
419 2 : side = Side.REVISION;
420 1 : } else if (mailComment.getInReplyTo() != null) {
421 1 : fileName = mailComment.getInReplyTo().key.filename;
422 1 : side = Side.fromShort(mailComment.getInReplyTo().side);
423 : } else {
424 1 : fileName = mailComment.getFileName();
425 1 : side = Side.REVISION;
426 : }
427 :
428 2 : HumanComment comment =
429 2 : commentsUtil.newHumanComment(
430 2 : ctx.getNotes(),
431 2 : ctx.getUser(),
432 2 : ctx.getWhen(),
433 : fileName,
434 2 : patchSetForComment.id(),
435 2 : (short) side.ordinal(),
436 2 : mailComment.getMessage(),
437 2 : false,
438 : null);
439 :
440 2 : comment.tag = tag;
441 2 : if (mailComment.getInReplyTo() != null) {
442 1 : comment.parentUuid = mailComment.getInReplyTo().key.uuid;
443 1 : comment.lineNbr = mailComment.getInReplyTo().lineNbr;
444 1 : comment.range = mailComment.getInReplyTo().range;
445 1 : comment.unresolved = mailComment.getInReplyTo().unresolved;
446 : }
447 2 : commentsUtil.setCommentCommitId(comment, ctx.getChange(), patchSetForComment);
448 2 : return comment;
449 : }
450 : }
451 :
452 : private static boolean useHtmlParser(MailMessage m) {
453 2 : return !Strings.isNullOrEmpty(m.htmlContent());
454 : }
455 :
456 : private static String numComments(int numComments) {
457 2 : return "(" + numComments + (numComments > 1 ? " comments)" : " comment)");
458 : }
459 :
460 : private Set<String> existingMessageIds(ChangeData cd) {
461 2 : Set<String> existingMessageIds = new HashSet<>();
462 2 : cd.messages().stream()
463 2 : .forEach(
464 : m -> {
465 2 : String messageId = CommentsUtil.extractMessageId(m.getTag());
466 2 : if (messageId != null) {
467 1 : existingMessageIds.add(messageId);
468 : }
469 2 : });
470 2 : cd.publishedComments().stream()
471 2 : .forEach(
472 : c -> {
473 2 : String messageId = CommentsUtil.extractMessageId(c.tag);
474 2 : if (messageId != null) {
475 1 : existingMessageIds.add(messageId);
476 : }
477 2 : });
478 2 : return existingMessageIds;
479 : }
480 : }
|
