Line data    Source code

       1             : // Copyright (C) 2011 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.rules;
      16             : 
      17             : import static com.googlecode.prolog_cafe.lang.PrologMachineCopy.save;
      18             : 
      19             : import com.google.common.base.Joiner;
      20             : import com.google.common.base.Strings;
      21             : import com.google.common.cache.Cache;
      22             : import com.google.common.collect.ImmutableList;
      23             : import com.google.gerrit.entities.Project;
      24             : import com.google.gerrit.entities.RefNames;
      25             : import com.google.gerrit.server.cache.CacheModule;
      26             : import com.google.gerrit.server.config.GerritServerConfig;
      27             : import com.google.gerrit.server.config.SitePaths;
      28             : import com.google.gerrit.server.git.GitRepositoryManager;
      29             : import com.google.gerrit.server.plugincontext.PluginSetContext;
      30             : import com.google.gerrit.server.project.ProjectCacheImpl;
      31             : import com.google.inject.Inject;
      32             : import com.google.inject.Singleton;
      33             : import com.google.inject.name.Named;
      34             : import com.googlecode.prolog_cafe.exceptions.CompileException;
      35             : import com.googlecode.prolog_cafe.exceptions.SyntaxException;
      36             : import com.googlecode.prolog_cafe.exceptions.TermException;
      37             : import com.googlecode.prolog_cafe.lang.BufferingPrologControl;
      38             : import com.googlecode.prolog_cafe.lang.JavaObjectTerm;
      39             : import com.googlecode.prolog_cafe.lang.ListTerm;
      40             : import com.googlecode.prolog_cafe.lang.Prolog;
      41             : import com.googlecode.prolog_cafe.lang.PrologClassLoader;
      42             : import com.googlecode.prolog_cafe.lang.PrologMachineCopy;
      43             : import com.googlecode.prolog_cafe.lang.StructureTerm;
      44             : import com.googlecode.prolog_cafe.lang.SymbolTerm;
      45             : import com.googlecode.prolog_cafe.lang.Term;
      46             : import java.io.IOException;
      47             : import java.io.PushbackReader;
      48             : import java.io.Reader;
      49             : import java.io.StringReader;
      50             : import java.net.MalformedURLException;
      51             : import java.net.URL;
      52             : import java.net.URLClassLoader;
      53             : import java.nio.file.Files;
      54             : import java.nio.file.Path;
      55             : import java.util.ArrayList;
      56             : import java.util.EnumSet;
      57             : import java.util.List;
      58             : import java.util.concurrent.ExecutionException;
      59             : import org.eclipse.jgit.annotations.Nullable;
      60             : import org.eclipse.jgit.errors.LargeObjectException;
      61             : import org.eclipse.jgit.lib.Config;
      62             : import org.eclipse.jgit.lib.Constants;
      63             : import org.eclipse.jgit.lib.ObjectId;
      64             : import org.eclipse.jgit.lib.ObjectLoader;
      65             : import org.eclipse.jgit.lib.Repository;
      66             : import org.eclipse.jgit.util.RawParseUtils;
      67             : 
      68             : /**
      69             :  * Manages a cache of compiled Prolog rules.
      70             :  *
      71             :  * <p>Rules are loaded from the {@code site_path/cache/rules/rules-SHA1.jar}, where {@code SHA1} is
      72             :  * the SHA1 of the Prolog {@code rules.pl} in a project's {@link RefNames#REFS_CONFIG} branch.
      73             :  */
      74             : @Singleton
      75             : public class RulesCache {
      76             :   public static class RulesCacheModule extends CacheModule {
      77             :     protected final Config config;
      78             : 
      79         152 :     public RulesCacheModule(Config config) {
      80         152 :       this.config = config;
      81         152 :     }
      82             : 
      83             :     @Override
      84             :     protected void configure() {
      85         152 :       if (has(ProjectCacheImpl.CACHE_NAME, "memoryLimit")) {
      86             :         // As this cache is auxiliary to the project cache, so size it the same when available
      87           0 :         cache(RulesCache.CACHE_NAME, ObjectId.class, PrologMachineCopy.class)
      88           0 :             .maximumWeight(config.getLong("cache", ProjectCacheImpl.CACHE_NAME, "memoryLimit", 0));
      89             :       } else {
      90         152 :         cache(RulesCache.CACHE_NAME, ObjectId.class, PrologMachineCopy.class);
      91             :       }
      92         152 :     }
      93             : 
      94             :     private boolean has(String name, String var) {
      95         152 :       return !Strings.isNullOrEmpty(config.getString("cache", name, var));
      96             :     }
      97             :   }
      98             : 
      99         146 :   private static final ImmutableList<String> PACKAGE_LIST =
     100         146 :       ImmutableList.of(Prolog.BUILTIN, "gerrit");
     101             : 
     102             :   static final String CACHE_NAME = "prolog_rules";
     103             : 
     104             :   private final boolean enableProjectRules;
     105             :   private final int maxDbSize;
     106             :   private final int compileReductionLimit;
     107             :   private final int maxSrcBytes;
     108             :   private final Path cacheDir;
     109             :   private final Path rulesDir;
     110             :   private final GitRepositoryManager gitMgr;
     111             :   private final PluginSetContext<PredicateProvider> predicateProviders;
     112             :   private final ClassLoader systemLoader;
     113             :   private final PrologMachineCopy defaultMachine;
     114             :   private final Cache<ObjectId, PrologMachineCopy> machineCache;
     115             : 
     116             :   @Inject
     117             :   protected RulesCache(
     118             :       @GerritServerConfig Config config,
     119             :       SitePaths site,
     120             :       GitRepositoryManager gm,
     121             :       PluginSetContext<PredicateProvider> predicateProviders,
     122         146 :       @Named(CACHE_NAME) Cache<ObjectId, PrologMachineCopy> machineCache) {
     123         146 :     maxDbSize = config.getInt("rules", null, "maxPrologDatabaseSize", 256);
     124         146 :     compileReductionLimit = RuleUtil.compileReductionLimit(config);
     125         146 :     maxSrcBytes = config.getInt("rules", null, "maxSourceBytes", 128 << 10);
     126         146 :     enableProjectRules = config.getBoolean("rules", null, "enable", true) && maxSrcBytes > 0;
     127         146 :     cacheDir = site.resolve(config.getString("cache", null, "directory"));
     128         146 :     rulesDir = cacheDir != null ? cacheDir.resolve("rules") : null;
     129         146 :     gitMgr = gm;
     130         146 :     this.predicateProviders = predicateProviders;
     131         146 :     this.machineCache = machineCache;
     132             : 
     133         146 :     systemLoader = getClass().getClassLoader();
     134         146 :     defaultMachine = save(newEmptyMachine(systemLoader));
     135         146 :   }
     136             : 
     137             :   public boolean isProjectRulesEnabled() {
     138           1 :     return enableProjectRules;
     139             :   }
     140             : 
     141             :   /**
     142             :    * Locate a cached Prolog machine state, or create one if not available.
     143             :    *
     144             :    * @return a Prolog machine, after loading the specified rules.
     145             :    * @throws CompileException the machine cannot be created.
     146             :    */
     147             :   public synchronized PrologMachineCopy loadMachine(
     148             :       @Nullable Project.NameKey project, @Nullable ObjectId rulesId) throws CompileException {
     149         103 :     if (!enableProjectRules || project == null || rulesId == null) {
     150         103 :       return defaultMachine;
     151             :     }
     152             : 
     153             :     try {
     154           5 :       return machineCache.get(rulesId, () -> createMachine(project, rulesId));
     155           1 :     } catch (ExecutionException e) {
     156           1 :       if (e.getCause() instanceof CompileException) {
     157           1 :         throw new CompileException(e.getCause().getMessage(), e);
     158             :       }
     159           0 :       throw new CompileException("Error while consulting rules from " + project, e);
     160             :     }
     161             :   }
     162             : 
     163             :   public PrologMachineCopy loadMachine(String name, Reader in) throws CompileException {
     164           1 :     PrologMachineCopy pmc = consultRules(name, in);
     165           1 :     if (pmc == null) {
     166           0 :       throw new CompileException("Cannot consult rules from the stream " + name);
     167             :     }
     168           1 :     return pmc;
     169             :   }
     170             : 
     171             :   private PrologMachineCopy createMachine(Project.NameKey project, ObjectId rulesId)
     172             :       throws CompileException {
     173             :     // If the rules are available as a complied JAR on local disk, prefer
     174             :     // that over dynamic consult as the bytecode will be faster.
     175             :     //
     176           5 :     if (rulesDir != null) {
     177           0 :       Path jarPath = rulesDir.resolve("rules-" + rulesId.getName() + ".jar");
     178           0 :       if (Files.isRegularFile(jarPath)) {
     179           0 :         URL[] cp = new URL[] {toURL(jarPath)};
     180           0 :         return save(newEmptyMachine(URLClassLoader.newInstance(cp, systemLoader)));
     181             :       }
     182             :     }
     183             : 
     184             :     // Dynamically consult the rules into the machine's internal database.
     185             :     //
     186           5 :     String rules = read(project, rulesId);
     187           5 :     PrologMachineCopy pmc = consultRules("rules.pl", new StringReader(rules));
     188           5 :     if (pmc == null) {
     189           0 :       throw new CompileException("Cannot consult rules of " + project);
     190             :     }
     191           5 :     return pmc;
     192             :   }
     193             : 
     194             :   @Nullable
     195             :   private PrologMachineCopy consultRules(String name, Reader rules) throws CompileException {
     196           5 :     BufferingPrologControl ctl = newEmptyMachine(systemLoader);
     197           5 :     PushbackReader in = new PushbackReader(rules, Prolog.PUSHBACK_SIZE);
     198             :     try {
     199           5 :       if (!ctl.execute(
     200           5 :           Prolog.BUILTIN, "consult_stream", SymbolTerm.intern(name), new JavaObjectTerm(in))) {
     201           0 :         return null;
     202             :       }
     203           0 :     } catch (SyntaxException e) {
     204           0 :       throw new CompileException(e.toString(), e);
     205           2 :     } catch (TermException e) {
     206           2 :       Term m = e.getMessageTerm();
     207           2 :       if (m instanceof StructureTerm && "syntax_error".equals(m.name()) && m.arity() >= 1) {
     208           2 :         StringBuilder msg = new StringBuilder();
     209           2 :         if (m.arg(0) instanceof ListTerm) {
     210           2 :           msg.append(Joiner.on(' ').join(((ListTerm) m.arg(0)).toJava()));
     211             :         } else {
     212           0 :           msg.append(m.arg(0).toString());
     213             :         }
     214           2 :         if (m.arity() == 2 && m.arg(1) instanceof StructureTerm && "at".equals(m.arg(1).name())) {
     215           2 :           Term at = m.arg(1).arg(0).dereference();
     216           2 :           if (at instanceof ListTerm) {
     217           2 :             msg.append(" at: ");
     218           2 :             msg.append(prettyProlog(at));
     219             :           }
     220             :         }
     221           2 :         throw new CompileException(msg.toString(), e);
     222             :       }
     223           0 :       throw new CompileException("Error while consulting rules from " + name, e);
     224           0 :     } catch (RuntimeException e) {
     225           0 :       throw new CompileException("Error while consulting rules from " + name, e);
     226           5 :     }
     227           5 :     return save(ctl);
     228             :   }
     229             : 
     230             :   private static String prettyProlog(Term at) {
     231           2 :     StringBuilder b = new StringBuilder();
     232           2 :     for (Object o : ((ListTerm) at).toJava()) {
     233           2 :       if (o instanceof Term) {
     234           2 :         Term t = (Term) o;
     235           2 :         if (!(t instanceof StructureTerm)) {
     236           0 :           b.append(t.toString()).append(' ');
     237           0 :           continue;
     238             :         }
     239           2 :         switch (t.name()) {
     240             :           case "atom":
     241           2 :             SymbolTerm atom = (SymbolTerm) t.arg(0);
     242           2 :             b.append(atom.toString());
     243           2 :             break;
     244             :           case "var":
     245           1 :             b.append(t.arg(0).toString());
     246             :             break;
     247             :         }
     248           2 :       } else {
     249           2 :         b.append(o);
     250             :       }
     251           2 :     }
     252           2 :     return b.toString().trim();
     253             :   }
     254             : 
     255             :   private String read(Project.NameKey project, ObjectId rulesId) throws CompileException {
     256           5 :     try (Repository git = gitMgr.openRepository(project)) {
     257             :       try {
     258           5 :         ObjectLoader ldr = git.open(rulesId, Constants.OBJ_BLOB);
     259           5 :         byte[] raw = ldr.getCachedBytes(maxSrcBytes);
     260           5 :         return RawParseUtils.decode(raw);
     261           0 :       } catch (LargeObjectException e) {
     262           0 :         throw new CompileException("rules of " + project + " are too large", e);
     263           0 :       } catch (RuntimeException | IOException e) {
     264           0 :         throw new CompileException("Cannot load rules of " + project, e);
     265             :       }
     266           0 :     } catch (IOException e) {
     267           0 :       throw new CompileException("Cannot open repository " + project, e);
     268             :     }
     269             :   }
     270             : 
     271             :   private BufferingPrologControl newEmptyMachine(ClassLoader cl) {
     272         146 :     BufferingPrologControl ctl = new BufferingPrologControl();
     273         146 :     ctl.setMaxDatabaseSize(maxDbSize);
     274             :     // Use the compiled reduction limit because the first term evaluation is done with
     275             :     // consult_stream - an internal, combined Prolog term.
     276         146 :     ctl.setReductionLimit(compileReductionLimit);
     277         146 :     ctl.setPrologClassLoader(
     278             :         new PrologClassLoader(new PredicateClassLoader(predicateProviders, cl)));
     279         146 :     ctl.setEnabled(EnumSet.allOf(Prolog.Feature.class), false);
     280             : 
     281         146 :     List<String> packages = new ArrayList<>();
     282         146 :     packages.addAll(PACKAGE_LIST);
     283         146 :     predicateProviders.runEach(
     284           0 :         predicateProvider -> packages.addAll(predicateProvider.getPackages()));
     285             : 
     286             :     // Bootstrap the interpreter and ensure there is clean state.
     287         146 :     ctl.initialize(packages.toArray(new String[packages.size()]));
     288         146 :     return ctl;
     289             :   }
     290             : 
     291             :   private static URL toURL(Path jarPath) throws CompileException {
     292             :     try {
     293           0 :       return jarPath.toUri().toURL();
     294           0 :     } catch (MalformedURLException e) {
     295           0 :       throw new CompileException("Cannot create URL for " + jarPath, e);
     296             :     }
     297             :   }
     298             : }