Line data    Source code

       1             : // Copyright (C) 2008 The Android Open Source Project
       2             : //
       3             : // Licensed under the Apache License, Version 2.0 (the "License");
       4             : // you may not use this file except in compliance with the License.
       5             : // You may obtain a copy of the License at
       6             : //
       7             : // http://www.apache.org/licenses/LICENSE-2.0
       8             : //
       9             : // Unless required by applicable law or agreed to in writing, software
      10             : // distributed under the License is distributed on an "AS IS" BASIS,
      11             : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12             : // See the License for the specific language governing permissions and
      13             : // limitations under the License.
      14             : 
      15             : package com.google.gerrit.server.git;
      16             : 
      17             : import com.google.common.flogger.FluentLogger;
      18             : import com.google.gerrit.entities.Project;
      19             : import com.google.gerrit.entities.Project.NameKey;
      20             : import com.google.gerrit.entities.RefNames;
      21             : import com.google.gerrit.extensions.events.LifecycleListener;
      22             : import com.google.gerrit.lifecycle.LifecycleModule;
      23             : import com.google.gerrit.server.config.GerritServerConfig;
      24             : import com.google.gerrit.server.config.SitePaths;
      25             : import com.google.inject.Inject;
      26             : import com.google.inject.Singleton;
      27             : import java.io.File;
      28             : import java.io.IOException;
      29             : import java.nio.file.FileVisitOption;
      30             : import java.nio.file.FileVisitResult;
      31             : import java.nio.file.Files;
      32             : import java.nio.file.Path;
      33             : import java.nio.file.SimpleFileVisitor;
      34             : import java.nio.file.attribute.BasicFileAttributes;
      35             : import java.util.Collections;
      36             : import java.util.EnumSet;
      37             : import java.util.Map;
      38             : import java.util.NavigableSet;
      39             : import java.util.TreeSet;
      40             : import java.util.concurrent.ConcurrentHashMap;
      41             : import org.eclipse.jgit.errors.RepositoryNotFoundException;
      42             : import org.eclipse.jgit.lib.Config;
      43             : import org.eclipse.jgit.lib.ConfigConstants;
      44             : import org.eclipse.jgit.lib.Constants;
      45             : import org.eclipse.jgit.lib.Repository;
      46             : import org.eclipse.jgit.lib.RepositoryCache;
      47             : import org.eclipse.jgit.lib.RepositoryCache.FileKey;
      48             : import org.eclipse.jgit.lib.RepositoryCacheConfig;
      49             : import org.eclipse.jgit.lib.StoredConfig;
      50             : import org.eclipse.jgit.storage.file.WindowCacheConfig;
      51             : import org.eclipse.jgit.util.FS;
      52             : 
      53             : /** Manages Git repositories stored on the local filesystem. */
      54             : @Singleton
      55             : public class LocalDiskRepositoryManager implements GitRepositoryManager {
      56          16 :   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
      57             : 
      58          15 :   public static class LocalDiskRepositoryManagerModule extends LifecycleModule {
      59             :     @Override
      60             :     protected void configure() {
      61          15 :       listener().to(LocalDiskRepositoryManager.Lifecycle.class);
      62          15 :     }
      63             :   }
      64             : 
      65             :   public static class Lifecycle implements LifecycleListener {
      66             :     private final Config serverConfig;
      67             : 
      68             :     @Inject
      69          15 :     Lifecycle(@GerritServerConfig Config cfg) {
      70          15 :       this.serverConfig = cfg;
      71          15 :     }
      72             : 
      73             :     @Override
      74             :     public void start() {
      75          15 :       RepositoryCacheConfig repoCacheCfg = new RepositoryCacheConfig();
      76          15 :       repoCacheCfg.fromConfig(serverConfig);
      77          15 :       repoCacheCfg.install();
      78             : 
      79          15 :       WindowCacheConfig cfg = new WindowCacheConfig();
      80          15 :       cfg.fromConfig(serverConfig);
      81          15 :       if (serverConfig.getString("core", null, "streamFileThreshold") == null) {
      82          15 :         long mx = Runtime.getRuntime().maxMemory();
      83          15 :         int limit =
      84             :             (int)
      85          15 :                 Math.min(
      86             :                     mx / 4, // don't use more than 1/4 of the heap.
      87             :                     2047 << 20); // cannot exceed array length
      88          15 :         if ((5 << 20) < limit && limit % (1 << 20) != 0) {
      89             :           // If the limit is at least 5 MiB but is not a whole multiple
      90             :           // of MiB round up to the next one full megabyte. This is a very
      91             :           // tiny memory increase in exchange for nice round units.
      92           0 :           limit = ((limit / (1 << 20)) + 1) << 20;
      93             :         }
      94             : 
      95             :         String desc;
      96          15 :         if (limit % (1 << 20) == 0) {
      97          15 :           desc = String.format("%dm", limit / (1 << 20));
      98           0 :         } else if (limit % (1 << 10) == 0) {
      99           0 :           desc = String.format("%dk", limit / (1 << 10));
     100             :         } else {
     101           0 :           desc = String.format("%d", limit);
     102             :         }
     103          15 :         logger.atInfo().log("Defaulting core.streamFileThreshold to %s", desc);
     104          15 :         cfg.setStreamFileThreshold(limit);
     105             :       }
     106          15 :       cfg.install();
     107          15 :     }
     108             : 
     109             :     @Override
     110          15 :     public void stop() {}
     111             :   }
     112             : 
     113             :   private final Path basePath;
     114          16 :   private final Map<Project.NameKey, FileKey> fileKeyByProject = new ConcurrentHashMap<>();
     115             : 
     116             :   @Inject
     117          16 :   LocalDiskRepositoryManager(SitePaths site, @GerritServerConfig Config cfg) {
     118          16 :     basePath = site.resolve(cfg.getString("gerrit", null, "basePath"));
     119          16 :     if (basePath == null) {
     120           1 :       throw new IllegalStateException("gerrit.basePath must be configured");
     121             :     }
     122          16 :   }
     123             : 
     124             :   /**
     125             :    * Return the basePath under which the specified project is stored.
     126             :    *
     127             :    * @param name the name of the project
     128             :    * @return base directory
     129             :    */
     130             :   public Path getBasePath(Project.NameKey name) {
     131          16 :     return basePath;
     132             :   }
     133             : 
     134             :   @Override
     135             :   public Status getRepositoryStatus(NameKey name) {
     136          13 :     if (isUnreasonableName(name)) {
     137           2 :       return Status.NON_EXISTENT;
     138             :     }
     139          13 :     Path path = getBasePath(name);
     140          13 :     File dir = FileKey.resolve(path.resolve(name.get()).toFile(), FS.DETECTED);
     141          13 :     if (dir == null) {
     142          13 :       return Status.NON_EXISTENT;
     143             :     }
     144             :     Repository repo;
     145             :     try {
     146             :       // Try to open with mustExist, so that it does not attempt to create a repository.
     147           1 :       repo = RepositoryCache.open(FileKey.lenient(dir, FS.DETECTED), /*mustExist=*/ true);
     148           0 :     } catch (RepositoryNotFoundException e) {
     149           0 :       return Status.NON_EXISTENT;
     150           0 :     } catch (IOException e) {
     151           0 :       return Status.UNAVAILABLE;
     152           1 :     }
     153             :     // If object database does not exist, the repository is unusable
     154           1 :     return repo.getObjectDatabase().exists() ? Status.ACTIVE : Status.UNAVAILABLE;
     155             :   }
     156             : 
     157             :   @Override
     158             :   public Repository openRepository(Project.NameKey name) throws RepositoryNotFoundException {
     159          16 :     FileKey cachedLocation = fileKeyByProject.get(name);
     160          16 :     if (cachedLocation != null) {
     161             :       try {
     162          15 :         return RepositoryCache.open(cachedLocation);
     163           0 :       } catch (IOException e) {
     164           0 :         fileKeyByProject.remove(name, cachedLocation);
     165             :       }
     166             :     }
     167             : 
     168          16 :     if (isUnreasonableName(name)) {
     169           2 :       throw new RepositoryNotFoundException("Invalid name: " + name);
     170             :     }
     171          16 :     FileKey location = FileKey.lenient(getBasePath(name).resolve(name.get()).toFile(), FS.DETECTED);
     172             :     try {
     173          16 :       Repository repo = RepositoryCache.open(location);
     174          16 :       fileKeyByProject.put(name, location);
     175          16 :       return repo;
     176          15 :     } catch (IOException e) {
     177          15 :       throw new RepositoryNotFoundException("Cannot open repository " + name, e);
     178             :     }
     179             :   }
     180             : 
     181             :   @Override
     182             :   public Repository createRepository(Project.NameKey name)
     183             :       throws RepositoryNotFoundException, RepositoryExistsException, IOException {
     184          16 :     if (isUnreasonableName(name)) {
     185           2 :       throw new RepositoryNotFoundException("Invalid name: " + name);
     186             :     }
     187             : 
     188          16 :     Path path = getBasePath(name);
     189          16 :     File dir = FileKey.resolve(path.resolve(name.get()).toFile(), FS.DETECTED);
     190          16 :     if (dir != null) {
     191             :       // Already exists on disk, use the repository we found.
     192             :       //
     193           1 :       Project.NameKey onDiskName = getProjectName(path, dir.getCanonicalFile().toPath());
     194             : 
     195           1 :       if (!onDiskName.equals(name)) {
     196           0 :         throw new RepositoryCaseMismatchException(name);
     197             :       }
     198           1 :       throw new RepositoryExistsException(name);
     199             :     }
     200             : 
     201             :     // It doesn't exist under any of the standard permutations
     202             :     // of the repository name, so prefer the standard bare name.
     203             :     //
     204          16 :     String n = name.get() + Constants.DOT_GIT_EXT;
     205          16 :     FileKey loc = FileKey.exact(path.resolve(n).toFile(), FS.DETECTED);
     206             : 
     207             :     try {
     208          16 :       Repository db = RepositoryCache.open(loc, false);
     209          16 :       db.create(true /* bare */);
     210             : 
     211          16 :       StoredConfig config = db.getConfig();
     212          16 :       config.setBoolean(
     213             :           ConfigConstants.CONFIG_CORE_SECTION,
     214             :           null,
     215             :           ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES,
     216             :           true);
     217          16 :       config.save();
     218             : 
     219             :       // JGit only writes to the reflog for refs/meta/config if the log file
     220             :       // already exists.
     221             :       //
     222          16 :       File metaConfigLog = new File(db.getDirectory(), "logs/" + RefNames.REFS_CONFIG);
     223          16 :       if (!metaConfigLog.getParentFile().mkdirs() || !metaConfigLog.createNewFile()) {
     224           0 :         logger.atSevere().log(
     225             :             "Failed to create ref log for %s in repository %s", RefNames.REFS_CONFIG, name);
     226             :       }
     227             : 
     228          16 :       return db;
     229           0 :     } catch (IOException e) {
     230           0 :       throw new RepositoryNotFoundException("Cannot create repository " + name, e);
     231             :     }
     232             :   }
     233             : 
     234             :   @Override
     235             :   public Boolean canPerformGC() {
     236          16 :     return true;
     237             :   }
     238             : 
     239             :   private boolean isUnreasonableName(Project.NameKey nameKey) {
     240          16 :     final String name = nameKey.get();
     241             : 
     242          16 :     return name.length() == 0 // no empty paths
     243          16 :         || name.charAt(name.length() - 1) == '/' // no suffix
     244          16 :         || name.indexOf('\\') >= 0 // no windows/dos style paths
     245          16 :         || name.charAt(0) == '/' // no absolute paths
     246          16 :         || new File(name).isAbsolute() // no absolute paths
     247          16 :         || name.startsWith("../") // no "l../etc/passwd"
     248          16 :         || name.contains("/../") // no "foo/../etc/passwd"
     249          16 :         || name.contains("/./") // "foo/./foo" is insane to ask
     250          16 :         || name.contains("//") // windows UNC path can be "//..."
     251          16 :         || name.contains(".git/") // no path segments that end with '.git' as "foo.git/bar"
     252          16 :         || name.contains("?") // common unix wildcard
     253          16 :         || name.contains("%") // wildcard or string parameter
     254          16 :         || name.contains("*") // wildcard
     255          16 :         || name.contains(":") // Could be used for absolute paths in windows?
     256          16 :         || name.contains("<") // redirect input
     257          16 :         || name.contains(">") // redirect output
     258          16 :         || name.contains("|") // pipe
     259          16 :         || name.contains("$") // dollar sign
     260          16 :         || name.contains("\r") // carriage return
     261          16 :         || name.contains("/+") // delimiter in /changes/
     262          16 :         || name.contains("~"); // delimiter in /changes/
     263             :   }
     264             : 
     265             :   @Override
     266             :   public NavigableSet<Project.NameKey> list() {
     267          16 :     ProjectVisitor visitor = new ProjectVisitor(basePath);
     268          16 :     scanProjects(visitor);
     269          16 :     return Collections.unmodifiableNavigableSet(visitor.found);
     270             :   }
     271             : 
     272             :   protected void scanProjects(ProjectVisitor visitor) {
     273             :     try {
     274          16 :       Files.walkFileTree(
     275             :           visitor.startFolder,
     276          16 :           EnumSet.of(FileVisitOption.FOLLOW_LINKS),
     277             :           Integer.MAX_VALUE,
     278             :           visitor);
     279           0 :     } catch (IOException e) {
     280           0 :       logger.atSevere().withCause(e).log(
     281           0 :           "Error walking repository tree %s", visitor.startFolder.toAbsolutePath());
     282          16 :     }
     283          16 :   }
     284             : 
     285             :   private static Project.NameKey getProjectName(Path startFolder, Path p) {
     286          16 :     String projectName = startFolder.relativize(p).toString();
     287          16 :     if (File.separatorChar != '/') {
     288           0 :       projectName = projectName.replace(File.separatorChar, '/');
     289             :     }
     290          16 :     if (projectName.endsWith(Constants.DOT_GIT_EXT)) {
     291          16 :       int newLen = projectName.length() - Constants.DOT_GIT_EXT.length();
     292          16 :       projectName = projectName.substring(0, newLen);
     293             :     }
     294          16 :     return Project.nameKey(projectName);
     295             :   }
     296             : 
     297             :   protected class ProjectVisitor extends SimpleFileVisitor<Path> {
     298          16 :     private final NavigableSet<Project.NameKey> found = new TreeSet<>();
     299             :     private Path startFolder;
     300             : 
     301          16 :     public ProjectVisitor(Path startFolder) {
     302          16 :       setStartFolder(startFolder);
     303          16 :     }
     304             : 
     305             :     public void setStartFolder(Path startFolder) {
     306          16 :       this.startFolder = startFolder;
     307          16 :     }
     308             : 
     309             :     @Override
     310             :     public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
     311             :         throws IOException {
     312          16 :       if (!dir.equals(startFolder) && isRepo(dir)) {
     313          16 :         addProject(dir);
     314          16 :         return FileVisitResult.SKIP_SUBTREE;
     315             :       }
     316          16 :       return FileVisitResult.CONTINUE;
     317             :     }
     318             : 
     319             :     @Override
     320             :     public FileVisitResult visitFileFailed(Path file, IOException e) {
     321           0 :       logger.atWarning().log("%s", e.getMessage());
     322           0 :       return FileVisitResult.CONTINUE;
     323             :     }
     324             : 
     325             :     private boolean isRepo(Path p) {
     326          16 :       String name = p.getFileName().toString();
     327          16 :       return !name.equals(Constants.DOT_GIT)
     328          16 :           && (name.endsWith(Constants.DOT_GIT_EXT)
     329          16 :               || FileKey.isGitRepository(p.toFile(), FS.DETECTED));
     330             :     }
     331             : 
     332             :     private void addProject(Path p) {
     333          16 :       Project.NameKey nameKey = getProjectName(startFolder, p);
     334          16 :       if (getBasePath(nameKey).equals(startFolder)) {
     335          16 :         if (isUnreasonableName(nameKey)) {
     336           1 :           logger.atWarning().log("Ignoring unreasonably named repository %s", p.toAbsolutePath());
     337             :         } else {
     338          16 :           found.add(nameKey);
     339             :         }
     340             :       }
     341          16 :     }
     342             :   }
     343             : }