Line data    Source code

       1             : // Copyright (C) 2020 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License
      14             : 
      15             : package com.google.gerrit.server.comment;
      16             : 
      17             : import static java.nio.charset.StandardCharsets.UTF_8;
      18             : 
      19             : import com.google.common.annotations.VisibleForTesting;
      20             : import com.google.common.cache.CacheLoader;
      21             : import com.google.common.cache.LoadingCache;
      22             : import com.google.common.cache.Weigher;
      23             : import com.google.common.collect.ImmutableList;
      24             : import com.google.common.collect.ImmutableMap;
      25             : import com.google.common.collect.Iterables;
      26             : import com.google.common.collect.Streams;
      27             : import com.google.common.flogger.FluentLogger;
      28             : import com.google.common.hash.Hashing;
      29             : import com.google.gerrit.entities.Change;
      30             : import com.google.gerrit.entities.Comment;
      31             : import com.google.gerrit.entities.CommentContext;
      32             : import com.google.gerrit.entities.HumanComment;
      33             : import com.google.gerrit.entities.Project;
      34             : import com.google.gerrit.exceptions.StorageException;
      35             : import com.google.gerrit.proto.Protos;
      36             : import com.google.gerrit.server.CommentsUtil;
      37             : import com.google.gerrit.server.cache.CacheModule;
      38             : import com.google.gerrit.server.cache.proto.Cache.AllCommentContextProto;
      39             : import com.google.gerrit.server.cache.proto.Cache.AllCommentContextProto.CommentContextProto;
      40             : import com.google.gerrit.server.cache.serialize.CacheSerializer;
      41             : import com.google.gerrit.server.comment.CommentContextLoader.ContextInput;
      42             : import com.google.gerrit.server.notedb.ChangeNotes;
      43             : import com.google.inject.Inject;
      44             : import com.google.inject.Module;
      45             : import com.google.inject.name.Named;
      46             : import java.io.IOException;
      47             : import java.util.HashMap;
      48             : import java.util.List;
      49             : import java.util.Map;
      50             : import java.util.concurrent.ExecutionException;
      51             : import java.util.function.Function;
      52             : import java.util.stream.Collectors;
      53             : 
      54             : /** Implementation of {@link CommentContextCache}. */
      55             : public class CommentContextCacheImpl implements CommentContextCache {
      56         152 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      57             : 
      58             :   private static final String CACHE_NAME = "comment_context";
      59             : 
      60             :   /**
      61             :    * Comment context is expected to contain just few lines of code to be displayed beside the
      62             :    * comment. Setting an upper bound of 100 for padding.
      63             :    */
      64             :   @VisibleForTesting public static final int MAX_CONTEXT_PADDING = 50;
      65             : 
      66             :   public static Module module() {
      67         152 :     return new CacheModule() {
      68             :       @Override
      69             :       protected void configure() {
      70         152 :         persist(CACHE_NAME, CommentContextKey.class, CommentContext.class)
      71         152 :             .version(5)
      72         152 :             .diskLimit(1 << 30) // limit the total cache size to 1 GB
      73         152 :             .maximumWeight(1 << 23) // Limit the size of the in-memory cache to 8 MB
      74         152 :             .weigher(CommentContextWeigher.class)
      75         152 :             .keySerializer(CommentContextKey.Serializer.INSTANCE)
      76         152 :             .valueSerializer(CommentContextSerializer.INSTANCE)
      77         152 :             .loader(Loader.class);
      78             : 
      79         152 :         bind(CommentContextCache.class).to(CommentContextCacheImpl.class);
      80         152 :       }
      81             :     };
      82             :   }
      83             : 
      84             :   private final LoadingCache<CommentContextKey, CommentContext> contextCache;
      85             : 
      86             :   @Inject
      87             :   CommentContextCacheImpl(
      88          24 :       @Named(CACHE_NAME) LoadingCache<CommentContextKey, CommentContext> contextCache) {
      89          24 :     this.contextCache = contextCache;
      90          24 :   }
      91             : 
      92             :   @Override
      93             :   public CommentContext get(CommentContextKey comment) {
      94           0 :     return getAll(ImmutableList.of(comment)).get(comment);
      95             :   }
      96             : 
      97             :   @Override
      98             :   public ImmutableMap<CommentContextKey, CommentContext> getAll(
      99             :       Iterable<CommentContextKey> inputKeys) {
     100           1 :     ImmutableMap.Builder<CommentContextKey, CommentContext> result = ImmutableMap.builder();
     101             : 
     102             :     // We do two transformations to the input keys: first we adjust the max context padding, and
     103             :     // second we hash the file path. The transformed keys are used to request context from the
     104             :     // cache. Keeping a map of the original inputKeys to the transformed keys
     105           1 :     Map<CommentContextKey, CommentContextKey> inputKeysToCacheKeys =
     106           1 :         Streams.stream(inputKeys)
     107           1 :             .collect(
     108           1 :                 Collectors.toMap(
     109           1 :                     Function.identity(),
     110             :                     k ->
     111           1 :                         adjustMaxContextPadding(k)
     112           1 :                             .toBuilder()
     113           1 :                             .path(Loader.hashPath(k.path()))
     114           1 :                             .build()));
     115             : 
     116             :     try {
     117           1 :       ImmutableMap<CommentContextKey, CommentContext> allContext =
     118           1 :           contextCache.getAll(inputKeysToCacheKeys.values());
     119             : 
     120           1 :       for (CommentContextKey inputKey : inputKeys) {
     121           1 :         CommentContextKey cacheKey = inputKeysToCacheKeys.get(inputKey);
     122           1 :         result.put(inputKey, allContext.get(cacheKey));
     123           1 :       }
     124           1 :       return result.build();
     125           0 :     } catch (ExecutionException e) {
     126           0 :       throw new StorageException("Failed to retrieve comments' context", e);
     127             :     }
     128             :   }
     129             : 
     130             :   private static CommentContextKey adjustMaxContextPadding(CommentContextKey key) {
     131           1 :     if (key.contextPadding() < 0) {
     132           0 :       logger.atWarning().log(
     133             :           "Cannot set context padding to a negative number %d. Adjusting the number to 0",
     134           0 :           key.contextPadding());
     135           0 :       return key.toBuilder().contextPadding(0).build();
     136             :     }
     137           1 :     if (key.contextPadding() > MAX_CONTEXT_PADDING) {
     138           1 :       logger.atWarning().log(
     139             :           "Number of requested context lines is %d and exceeding the configured maximum of %d."
     140             :               + " Adjusting the number to the maximum.",
     141           1 :           key.contextPadding(), MAX_CONTEXT_PADDING);
     142           1 :       return key.toBuilder().contextPadding(MAX_CONTEXT_PADDING).build();
     143             :     }
     144           1 :     return key;
     145             :   }
     146             : 
     147         153 :   public enum CommentContextSerializer implements CacheSerializer<CommentContext> {
     148         153 :     INSTANCE;
     149             : 
     150             :     @Override
     151             :     public byte[] serialize(CommentContext commentContext) {
     152           1 :       AllCommentContextProto.Builder allBuilder = AllCommentContextProto.newBuilder();
     153           1 :       allBuilder.setContentType(commentContext.contentType());
     154             : 
     155           1 :       commentContext
     156           1 :           .lines()
     157           1 :           .entrySet()
     158           1 :           .forEach(
     159             :               c ->
     160           1 :                   allBuilder.addContext(
     161           1 :                       CommentContextProto.newBuilder()
     162           1 :                           .setLineNumber(c.getKey())
     163           1 :                           .setContextLine(c.getValue())));
     164           1 :       return Protos.toByteArray(allBuilder.build());
     165             :     }
     166             : 
     167             :     @Override
     168             :     public CommentContext deserialize(byte[] in) {
     169           1 :       ImmutableMap.Builder<Integer, String> contextLinesMap = ImmutableMap.builder();
     170           1 :       AllCommentContextProto proto = Protos.parseUnchecked(AllCommentContextProto.parser(), in);
     171           1 :       proto.getContextList().stream()
     172           1 :           .forEach(c -> contextLinesMap.put(c.getLineNumber(), c.getContextLine()));
     173           1 :       return CommentContext.create(contextLinesMap.build(), proto.getContentType());
     174             :     }
     175             :   }
     176             : 
     177             :   static class Loader extends CacheLoader<CommentContextKey, CommentContext> {
     178             :     private final ChangeNotes.Factory notesFactory;
     179             :     private final CommentsUtil commentsUtil;
     180             :     private final CommentContextLoader.Factory factory;
     181             : 
     182             :     @Inject
     183             :     Loader(
     184             :         CommentsUtil commentsUtil,
     185             :         ChangeNotes.Factory notesFactory,
     186         152 :         CommentContextLoader.Factory factory) {
     187         152 :       this.commentsUtil = commentsUtil;
     188         152 :       this.notesFactory = notesFactory;
     189         152 :       this.factory = factory;
     190         152 :     }
     191             : 
     192             :     /**
     193             :      * Load the comment context of a single comment identified by its key.
     194             :      *
     195             :      * @param key a {@link CommentContextKey} identifying a comment.
     196             :      * @return the comment context associated with the comment.
     197             :      * @throws IOException an error happened while parsing the commit or loading the file where the
     198             :      *     comment is written.
     199             :      */
     200             :     @Override
     201             :     public CommentContext load(CommentContextKey key) throws IOException {
     202           0 :       return loadAll(ImmutableList.of(key)).get(key);
     203             :     }
     204             : 
     205             :     /**
     206             :      * Load the comment context of different comments identified by their keys.
     207             :      *
     208             :      * @param keys list of {@link CommentContextKey} identifying some comments.
     209             :      * @return a map of the input keys to their corresponding comment context.
     210             :      * @throws IOException an error happened while parsing the commits or loading the files where
     211             :      *     the comments are written.
     212             :      */
     213             :     @Override
     214             :     public Map<CommentContextKey, CommentContext> loadAll(
     215             :         Iterable<? extends CommentContextKey> keys) throws IOException {
     216           1 :       ImmutableMap.Builder<CommentContextKey, CommentContext> result =
     217           1 :           ImmutableMap.builderWithExpectedSize(Iterables.size(keys));
     218             : 
     219           1 :       Map<Project.NameKey, Map<Change.Id, List<CommentContextKey>>> groupedKeys =
     220           1 :           Streams.stream(keys)
     221           1 :               .distinct()
     222           1 :               .map(k -> (CommentContextKey) k)
     223           1 :               .collect(
     224           1 :                   Collectors.groupingBy(
     225             :                       CommentContextKey::project,
     226           1 :                       Collectors.groupingBy(CommentContextKey::changeId)));
     227             : 
     228             :       for (Map.Entry<Project.NameKey, Map<Change.Id, List<CommentContextKey>>> perProject :
     229           1 :           groupedKeys.entrySet()) {
     230           1 :         Map<Change.Id, List<CommentContextKey>> keysPerProject = perProject.getValue();
     231             : 
     232           1 :         for (Map.Entry<Change.Id, List<CommentContextKey>> perChange : keysPerProject.entrySet()) {
     233           1 :           Map<CommentContextKey, CommentContext> context =
     234           1 :               loadForSameChange(perChange.getValue(), perProject.getKey(), perChange.getKey());
     235           1 :           result.putAll(context);
     236           1 :         }
     237           1 :       }
     238           1 :       return result.build();
     239             :     }
     240             : 
     241             :     /**
     242             :      * Load the comment context for comments (published and drafts) of the same project and change
     243             :      * ID.
     244             :      *
     245             :      * @param keys a list of keys corresponding to some comments
     246             :      * @param project a gerrit project/repository
     247             :      * @param changeId an identifier for a change
     248             :      * @return a map of the input keys to their corresponding {@link CommentContext}
     249             :      */
     250             :     private Map<CommentContextKey, CommentContext> loadForSameChange(
     251             :         List<CommentContextKey> keys, Project.NameKey project, Change.Id changeId)
     252             :         throws IOException {
     253           1 :       ChangeNotes notes = notesFactory.createChecked(project, changeId);
     254           1 :       List<HumanComment> humanComments = commentsUtil.publishedHumanCommentsByChange(notes);
     255           1 :       List<HumanComment> drafts = commentsUtil.draftByChange(notes);
     256           1 :       List<HumanComment> allComments =
     257           1 :           Streams.concat(humanComments.stream(), drafts.stream()).collect(Collectors.toList());
     258           1 :       CommentContextLoader loader = factory.create(project);
     259           1 :       Map<CommentContextKey, ContextInput> keysToComments = new HashMap<>();
     260           1 :       for (CommentContextKey key : keys) {
     261           1 :         Comment comment = getCommentForKey(allComments, key);
     262           1 :         keysToComments.put(key, ContextInput.fromComment(comment, key.contextPadding()));
     263           1 :       }
     264           1 :       Map<ContextInput, CommentContext> allContext =
     265           1 :           loader.getContext(
     266           1 :               keysToComments.values().stream().distinct().collect(Collectors.toList()));
     267           1 :       return keys.stream()
     268           1 :           .collect(
     269           1 :               Collectors.toMap(Function.identity(), k -> allContext.get(keysToComments.get(k))));
     270             :     }
     271             : 
     272             :     /**
     273             :      * Return the single comment from the {@code allComments} input list corresponding to the key
     274             :      * parameter.
     275             :      *
     276             :      * @param allComments a list of comments.
     277             :      * @param key a key representing a single comment.
     278             :      * @return the single comment corresponding to the key parameter.
     279             :      */
     280             :     private Comment getCommentForKey(List<HumanComment> allComments, CommentContextKey key) {
     281           1 :       return allComments.stream()
     282           1 :           .filter(
     283             :               c ->
     284           1 :                   key.id().equals(c.key.uuid)
     285           1 :                       && key.patchset() == c.key.patchSetId
     286           1 :                       && key.path().equals(hashPath(c.key.filename)))
     287           1 :           .findFirst()
     288           1 :           .orElseThrow(() -> new IllegalArgumentException("Unable to find comment for key " + key));
     289             :     }
     290             : 
     291             :     /**
     292             :      * Hash an input String using the general {@link Hashing#murmur3_128()} hash.
     293             :      *
     294             :      * @param input the input String
     295             :      * @return a hashed representation of the input String
     296             :      */
     297             :     static String hashPath(String input) {
     298           1 :       return Hashing.murmur3_128().hashString(input, UTF_8).toString();
     299             :     }
     300             :   }
     301             : 
     302             :   private static class CommentContextWeigher implements Weigher<CommentContextKey, CommentContext> {
     303             :     @Override
     304             :     public int weigh(CommentContextKey key, CommentContext commentContext) {
     305           1 :       int size = 0;
     306           1 :       size += key.id().length();
     307           1 :       size += key.path().length();
     308           1 :       size += key.project().get().length();
     309           1 :       size += 4;
     310           1 :       for (String line : commentContext.lines().values()) {
     311           1 :         size += 4; // line number
     312           1 :         size += line.length(); // number of characters in the context line
     313           1 :       }
     314           1 :       return size;
     315             :     }
     316             :   }
     317             : }