Line data Source code
1 : // Copyright (C) 2013 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.pgm;
16 :
17 : import static java.util.Objects.requireNonNull;
18 : import static java.util.stream.Collectors.toSet;
19 :
20 : import com.google.common.cache.Cache;
21 : import com.google.common.collect.Sets;
22 : import com.google.gerrit.common.Die;
23 : import com.google.gerrit.extensions.config.FactoryModule;
24 : import com.google.gerrit.extensions.registration.DynamicMap;
25 : import com.google.gerrit.index.Index;
26 : import com.google.gerrit.index.IndexDefinition;
27 : import com.google.gerrit.index.IndexType;
28 : import com.google.gerrit.index.SiteIndexer;
29 : import com.google.gerrit.lifecycle.LifecycleManager;
30 : import com.google.gerrit.lucene.LuceneIndexModule;
31 : import com.google.gerrit.pgm.util.BatchProgramModule;
32 : import com.google.gerrit.pgm.util.SiteProgram;
33 : import com.google.gerrit.server.LibModuleLoader;
34 : import com.google.gerrit.server.ModuleOverloader;
35 : import com.google.gerrit.server.cache.CacheDisplay;
36 : import com.google.gerrit.server.cache.CacheInfo;
37 : import com.google.gerrit.server.change.ChangeResource;
38 : import com.google.gerrit.server.config.GerritServerConfig;
39 : import com.google.gerrit.server.git.WorkQueue.WorkQueueModule;
40 : import com.google.gerrit.server.index.IndexModule;
41 : import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
42 : import com.google.gerrit.server.index.options.AutoFlush;
43 : import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
44 : import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
45 : import com.google.gerrit.server.util.ReplicaUtil;
46 : import com.google.inject.AbstractModule;
47 : import com.google.inject.Inject;
48 : import com.google.inject.Injector;
49 : import com.google.inject.Key;
50 : import com.google.inject.Module;
51 : import com.google.inject.multibindings.OptionalBinder;
52 : import java.io.StringWriter;
53 : import java.io.Writer;
54 : import java.lang.reflect.InvocationTargetException;
55 : import java.lang.reflect.Method;
56 : import java.util.ArrayList;
57 : import java.util.Collection;
58 : import java.util.HashMap;
59 : import java.util.List;
60 : import java.util.Map;
61 : import java.util.Set;
62 : import java.util.TreeSet;
63 : import java.util.concurrent.TimeUnit;
64 : import java.util.stream.Collectors;
65 : import java.util.stream.StreamSupport;
66 : import org.eclipse.jgit.lib.Config;
67 : import org.eclipse.jgit.util.io.NullOutputStream;
68 : import org.kohsuke.args4j.Option;
69 :
70 15 : public class Reindex extends SiteProgram {
71 15 : @Option(
72 : name = "--threads",
73 : usage = "Number of threads to use for indexing. Default is index.batchThreads from config.")
74 : private int threads = 0;
75 :
76 : @Option(
77 : name = "--changes-schema-version",
78 : usage = "Schema version to reindex, for changes; default is most recent version")
79 : private Integer changesVersion;
80 :
81 : @Option(name = "--verbose", usage = "Output debug information for each change")
82 : private boolean verbose;
83 :
84 : @Option(name = "--list", usage = "List supported indices and exit")
85 : private boolean list;
86 :
87 15 : @Option(name = "--index", usage = "Only reindex specified indices")
88 : private List<String> indices = new ArrayList<>();
89 :
90 : @Option(
91 : name = "--disable-cache-stats",
92 : usage =
93 : "Disables printing the cache statistics."
94 : + "Defaults to true when reindex is run from init on a new site, false otherwise")
95 : private boolean disableCacheStats;
96 :
97 : private Injector dbInjector;
98 : private Injector sysInjector;
99 : private Injector cfgInjector;
100 : private Config globalConfig;
101 :
102 : @Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs;
103 : @Inject private DynamicMap<Cache<?, ?>> cacheMap;
104 :
105 : @Override
106 : public int run() throws Exception {
107 15 : mustHaveValidSite();
108 15 : dbInjector = createDbInjector();
109 15 : cfgInjector = dbInjector.createChildInjector();
110 15 : globalConfig = dbInjector.getInstance(Key.get(Config.class, GerritServerConfig.class));
111 15 : overrideConfig();
112 15 : LifecycleManager dbManager = new LifecycleManager();
113 15 : dbManager.add(dbInjector);
114 15 : dbManager.start();
115 :
116 15 : sysInjector = createSysInjector();
117 15 : sysInjector.getInstance(PluginGuiceEnvironment.class).setDbCfgInjector(dbInjector, cfgInjector);
118 15 : LifecycleManager sysManager = new LifecycleManager();
119 15 : sysManager.add(sysInjector);
120 15 : sysManager.start();
121 :
122 15 : sysInjector.injectMembers(this);
123 15 : checkIndicesOption();
124 :
125 : try {
126 15 : boolean ok = list ? list() : reindex();
127 15 : if (!disableCacheStats) {
128 15 : printCacheStats();
129 : }
130 15 : return ok ? 0 : 1;
131 0 : } catch (Exception e) {
132 0 : throw die(e.getMessage(), e);
133 : } finally {
134 15 : sysManager.stop();
135 15 : dbManager.stop();
136 : }
137 : }
138 :
139 : private boolean list() {
140 0 : for (IndexDefinition<?, ?, ?> def : indexDefs) {
141 0 : System.out.format("%s\n", def.getName());
142 0 : }
143 0 : return true;
144 : }
145 :
146 : private boolean reindex() {
147 15 : boolean ok = true;
148 15 : for (IndexDefinition<?, ?, ?> def : indexDefs) {
149 15 : if (indices.isEmpty() || indices.contains(def.getName())) {
150 15 : ok &= reindex(def);
151 : }
152 15 : }
153 15 : return ok;
154 : }
155 :
156 : private void checkIndicesOption() throws Die {
157 15 : if (indices.isEmpty()) {
158 1 : return;
159 : }
160 :
161 15 : requireNonNull(indexDefs, "Called this method before injectMembers?");
162 15 : Set<String> valid = indexDefs.stream().map(IndexDefinition::getName).sorted().collect(toSet());
163 15 : Set<String> invalid = Sets.difference(Sets.newHashSet(indices), valid);
164 15 : if (invalid.isEmpty()) {
165 15 : return;
166 : }
167 :
168 1 : throw die(
169 : "invalid index name(s): " + new TreeSet<>(invalid) + " available indices are: " + valid);
170 : }
171 :
172 : private Injector createSysInjector() {
173 15 : Map<String, Integer> versions = new HashMap<>();
174 15 : if (changesVersion != null) {
175 0 : versions.put(ChangeSchemaDefinitions.INSTANCE.getName(), changesVersion);
176 : }
177 15 : boolean replica = ReplicaUtil.isReplica(globalConfig);
178 15 : List<Module> modules = new ArrayList<>();
179 15 : modules.add(new WorkQueueModule());
180 :
181 : Module indexModule;
182 15 : IndexType indexType = IndexModule.getIndexType(dbInjector);
183 15 : if (indexType.isLucene()) {
184 1 : indexModule =
185 1 : LuceneIndexModule.singleVersionWithExplicitVersions(
186 : versions, threads, replica, AutoFlush.DISABLED);
187 15 : } else if (indexType.isFake()) {
188 : // Use Reflection so that we can omit the fake index binary in production code. Test code does
189 : // compile the component in.
190 : try {
191 15 : Class<?> clazz = Class.forName("com.google.gerrit.index.testing.FakeIndexModule");
192 15 : Method m =
193 15 : clazz.getMethod(
194 : "singleVersionWithExplicitVersions", Map.class, int.class, boolean.class);
195 15 : indexModule = (Module) m.invoke(null, versions, threads, replica);
196 0 : } catch (NoSuchMethodException
197 : | ClassNotFoundException
198 : | IllegalAccessException
199 : | InvocationTargetException e) {
200 0 : throw new IllegalStateException("can't create index", e);
201 15 : }
202 : } else {
203 0 : throw new IllegalStateException("unsupported index.type = " + indexType);
204 : }
205 15 : modules.add(indexModule);
206 15 : modules.add(
207 15 : new AbstractModule() {
208 : @Override
209 : protected void configure() {
210 15 : super.configure();
211 15 : OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
212 15 : .setBinding()
213 15 : .toInstance(IsFirstInsertForEntry.YES);
214 15 : }
215 : });
216 15 : modules.add(new BatchProgramModule(dbInjector));
217 15 : modules.add(
218 15 : new FactoryModule() {
219 : @Override
220 : protected void configure() {
221 15 : factory(ChangeResource.Factory.class);
222 15 : }
223 : });
224 :
225 15 : return dbInjector.createChildInjector(
226 15 : ModuleOverloader.override(
227 15 : modules, LibModuleLoader.loadReindexModules(cfgInjector, versions, threads, replica)));
228 : }
229 :
230 : private void overrideConfig() {
231 : // Disable auto-commit for speed; committing will happen at the end of the process.
232 15 : if (IndexModule.getIndexType(dbInjector).isLucene()) {
233 1 : globalConfig.setLong("index", "changes_open", "commitWithin", -1);
234 1 : globalConfig.setLong("index", "changes_closed", "commitWithin", -1);
235 : }
236 :
237 : // Disable change cache.
238 15 : globalConfig.setLong("cache", "changes", "maximumWeight", 0);
239 :
240 : // Disable auto-reindexing if stale, since there are no concurrent writes to race with.
241 15 : globalConfig.setBoolean("index", null, "autoReindexIfStale", false);
242 15 : }
243 :
244 : private <K, V, I extends Index<K, V>> boolean reindex(IndexDefinition<K, V, I> def) {
245 15 : I index = def.getIndexCollection().getSearchIndex();
246 15 : requireNonNull(
247 0 : index, () -> String.format("no active search index configured for %s", def.getName()));
248 15 : index.markReady(false);
249 15 : index.deleteAll();
250 :
251 15 : SiteIndexer<K, V, I> siteIndexer = def.getSiteIndexer();
252 15 : siteIndexer.setProgressOut(System.err);
253 15 : siteIndexer.setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE);
254 15 : SiteIndexer.Result result = siteIndexer.indexAll(index);
255 15 : int n = result.doneCount() + result.failedCount();
256 15 : double t = result.elapsed(TimeUnit.MILLISECONDS) / 1000d;
257 15 : System.out.format(
258 15 : "Reindexed %d documents in %s index in %.01fs (%.01f/s)\n", n, def.getName(), t, n / t);
259 15 : if (result.success()) {
260 15 : index.markReady(true);
261 : }
262 15 : System.out.format(
263 : "Index %s in version %d is %sready\n",
264 15 : def.getName(), index.getSchema().getVersion(), result.success() ? "" : "NOT ");
265 :
266 15 : return result.success();
267 : }
268 :
269 : private void printCacheStats() {
270 15 : try (Writer sw = new StringWriter()) {
271 15 : sw.write("Cache Statistics at the end of reindexing\n");
272 15 : new CacheDisplay(
273 : sw,
274 15 : StreamSupport.stream(cacheMap.spliterator(), false)
275 15 : .map(e -> new CacheInfo(e.getExportName(), e.get()))
276 15 : .collect(Collectors.toList()))
277 15 : .displayCaches();
278 15 : System.out.print(sw.toString());
279 0 : } catch (Exception e) {
280 0 : System.out.format("Error displaying the cache statistics\n" + e.getMessage());
281 15 : }
282 15 : }
283 : }
|
