Line data Source code
1 : // Copyright (C) 2022 The Android Open Source Project
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : package com.google.gerrit.index;
16 :
17 : import static com.google.common.base.Preconditions.checkArgument;
18 : import static com.google.common.base.Preconditions.checkState;
19 : import static com.google.common.collect.ImmutableList.toImmutableList;
20 :
21 : import com.google.auto.value.AutoValue;
22 : import com.google.common.base.CharMatcher;
23 : import com.google.common.collect.ImmutableList;
24 : import com.google.common.collect.ImmutableMap;
25 : import com.google.common.reflect.TypeToken;
26 : import com.google.gerrit.common.Nullable;
27 : import com.google.gerrit.entities.converter.ProtoConverter;
28 : import com.google.gerrit.exceptions.StorageException;
29 : import com.google.gerrit.index.SchemaFieldDefs.Getter;
30 : import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
31 : import com.google.gerrit.index.SchemaFieldDefs.Setter;
32 : import com.google.gerrit.proto.Protos;
33 : import com.google.protobuf.MessageLite;
34 : import java.io.IOException;
35 : import java.lang.reflect.ParameterizedType;
36 : import java.lang.reflect.Type;
37 : import java.sql.Timestamp;
38 : import java.util.HashMap;
39 : import java.util.Map;
40 : import java.util.Optional;
41 : import java.util.stream.StreamSupport;
42 :
43 : /**
44 : * Definition of a field stored in the secondary index.
45 : *
46 : * <p>Each IndexedField, stored in index, may have multiple {@link SearchSpec} which defines how it
47 : * can be searched and how the index tokens are generated.
48 : *
49 : * <p>Index implementations may choose to store IndexedField and {@link SearchSpec} (search tokens)
50 : * separately, however {@link com.google.gerrit.index.query.IndexedQuery} always issues the queries
51 : * to {@link SearchSpec}.
52 : *
53 : * <p>This allows index implementations to store IndexedField once, while enabling multiple
54 : * tokenization strategies on the same IndexedField with {@link SearchSpec}
55 : *
56 : * @param <I> input type from which documents are created and search results are returned.
57 : * @param <T> type that should be extracted from the input object when converting to an index
58 : * document.
59 : */
60 : // TODO(mariasavtchouk): revisit the class name after migration is done.
61 : @SuppressWarnings("serial")
62 : @AutoValue
63 155 : public abstract class IndexedField<I, T> {
64 :
65 155 : public static final TypeToken<Integer> INTEGER_TYPE = new TypeToken<>() {};
66 155 : public static final TypeToken<Iterable<Integer>> ITERABLE_INTEGER_TYPE = new TypeToken<>() {};
67 155 : public static final TypeToken<Long> LONG_TYPE = new TypeToken<>() {};
68 155 : public static final TypeToken<Iterable<Long>> ITERABLE_LONG_TYPE = new TypeToken<>() {};
69 155 : public static final TypeToken<String> STRING_TYPE = new TypeToken<>() {};
70 155 : public static final TypeToken<Iterable<String>> ITERABLE_STRING_TYPE = new TypeToken<>() {};
71 155 : public static final TypeToken<byte[]> BYTE_ARRAY_TYPE = new TypeToken<>() {};
72 155 : public static final TypeToken<Iterable<byte[]>> ITERABLE_BYTE_ARRAY_TYPE = new TypeToken<>() {};
73 155 : public static final TypeToken<Timestamp> TIMESTAMP_TYPE = new TypeToken<>() {};
74 :
75 : // Should not be used directly, only used to check if the proto is stored
76 155 : private static final TypeToken<MessageLite> MESSAGE_TYPE = new TypeToken<>() {};
77 :
78 : public static <I, T> Builder<I, T> builder(String name, TypeToken<T> fieldType) {
79 155 : return new AutoValue_IndexedField.Builder<I, T>()
80 155 : .name(name)
81 155 : .fieldType(fieldType)
82 155 : .stored(false)
83 155 : .required(false);
84 : }
85 :
86 : public static <I> Builder<I, Iterable<String>> iterableStringBuilder(String name) {
87 155 : return builder(name, IndexedField.ITERABLE_STRING_TYPE);
88 : }
89 :
90 : public static <I> Builder<I, String> stringBuilder(String name) {
91 155 : return builder(name, IndexedField.STRING_TYPE);
92 : }
93 :
94 : public static <I> Builder<I, Integer> integerBuilder(String name) {
95 155 : return builder(name, IndexedField.INTEGER_TYPE);
96 : }
97 :
98 : public static <I> Builder<I, Long> longBuilder(String name) {
99 1 : return builder(name, IndexedField.LONG_TYPE);
100 : }
101 :
102 : public static <I> Builder<I, Iterable<Integer>> iterableIntegerBuilder(String name) {
103 155 : return builder(name, IndexedField.ITERABLE_INTEGER_TYPE);
104 : }
105 :
106 : public static <I> Builder<I, Timestamp> timestampBuilder(String name) {
107 155 : return builder(name, IndexedField.TIMESTAMP_TYPE);
108 : }
109 :
110 : public static <I> Builder<I, byte[]> byteArrayBuilder(String name) {
111 153 : return builder(name, IndexedField.BYTE_ARRAY_TYPE);
112 : }
113 :
114 : public static <I> Builder<I, Iterable<byte[]>> iterableByteArrayBuilder(String name) {
115 155 : return builder(name, IndexedField.ITERABLE_BYTE_ARRAY_TYPE);
116 : }
117 :
118 : /**
119 : * Defines how {@link IndexedField} can be searched and how the index tokens are generated.
120 : *
121 : * <p>Multiple {@link SearchSpec} can be defined on a single {@link IndexedField}.
122 : *
123 : * <p>Depending on the implementation, indexes can choose to store {@link IndexedField} and {@link
124 : * SearchSpec} separately. The searches are issues to {@link SearchSpec}.
125 : */
126 : public class SearchSpec implements SchemaField<I, T> {
127 : private final String name;
128 : private final SearchOption searchOption;
129 :
130 155 : public SearchSpec(String name, SearchOption searchOption) {
131 155 : checkName(name);
132 155 : this.name = name;
133 155 : this.searchOption = searchOption;
134 155 : }
135 :
136 : @Override
137 : public boolean isStored() {
138 154 : return getField().stored();
139 : }
140 :
141 : @Override
142 : public boolean isRepeatable() {
143 151 : return getField().repeatable();
144 : }
145 :
146 : @Override
147 : @Nullable
148 : public T get(I obj) {
149 151 : return getField().get(obj);
150 : }
151 :
152 : @Override
153 : public String getName() {
154 155 : return name;
155 : }
156 :
157 : @Override
158 : public FieldType<?> getType() {
159 151 : SearchOption searchOption = getSearchOption();
160 151 : TypeToken<?> fieldType = getField().fieldType();
161 151 : if (searchOption.equals(SearchOption.STORE_ONLY)) {
162 7 : return FieldType.STORED_ONLY;
163 151 : } else if ((fieldType.equals(IndexedField.INTEGER_TYPE)
164 151 : || fieldType.equals(IndexedField.ITERABLE_INTEGER_TYPE))
165 78 : && searchOption.equals(SearchOption.EXACT)) {
166 78 : return FieldType.INTEGER;
167 151 : } else if (fieldType.equals(IndexedField.INTEGER_TYPE)
168 4 : && searchOption.equals(SearchOption.RANGE)) {
169 4 : return FieldType.INTEGER_RANGE;
170 151 : } else if (fieldType.equals(IndexedField.LONG_TYPE)) {
171 0 : return FieldType.LONG;
172 151 : } else if (fieldType.equals(IndexedField.TIMESTAMP_TYPE)) {
173 7 : return FieldType.TIMESTAMP;
174 151 : } else if (fieldType.equals(IndexedField.STRING_TYPE)
175 40 : || fieldType.equals(IndexedField.ITERABLE_STRING_TYPE)) {
176 151 : if (searchOption.equals(SearchOption.EXACT)) {
177 151 : return FieldType.EXACT;
178 36 : } else if (searchOption.equals(SearchOption.FULL_TEXT)) {
179 11 : return FieldType.FULL_TEXT;
180 36 : } else if (searchOption.equals(SearchOption.PREFIX)) {
181 36 : return FieldType.PREFIX;
182 : }
183 : }
184 0 : throw new IllegalArgumentException(
185 0 : String.format(
186 : "search spec [%s, %s] is not supported on field [%s, %s]",
187 0 : getName(), getSearchOption(), getField().name(), getField().fieldType()));
188 : }
189 :
190 : @Override
191 : public boolean setIfPossible(I object, StoredValue doc) {
192 101 : return getField().setIfPossible(object, doc);
193 : }
194 :
195 : /**
196 : * Returns {@link SearchOption} enabled on this field.
197 : *
198 : * @return {@link SearchOption}
199 : */
200 : public SearchOption getSearchOption() {
201 151 : return searchOption;
202 : }
203 :
204 : /**
205 : * Returns {@link IndexedField} on which this spec was created.
206 : *
207 : * @return original {@link IndexedField} of this spec.
208 : */
209 : public IndexedField<I, T> getField() {
210 154 : return IndexedField.this;
211 : }
212 :
213 : private String checkName(String name) {
214 155 : CharMatcher m = CharMatcher.anyOf("abcdefghijklmnopqrstuvwxyz0123456789_");
215 155 : checkArgument(name != null && m.matchesAllOf(name), "illegal field name: %s", name);
216 155 : return name;
217 : }
218 : }
219 :
220 : /**
221 : * Adds {@link SearchSpec} to this {@link IndexedField}
222 : *
223 : * @param name the name to use for in the search.
224 : * @param searchOption the tokenization option, enabled by the new {@link SearchSpec}
225 : * @return the added {@link SearchSpec}.
226 : */
227 : public SearchSpec addSearchSpec(String name, SearchOption searchOption) {
228 155 : SearchSpec searchSpec = new SearchSpec(name, searchOption);
229 155 : checkArgument(
230 155 : !searchSpecs.containsKey(searchSpec.getName()),
231 : "Can not add search spec %s, because it is already defined on field %s",
232 155 : searchSpec.getName(),
233 155 : name());
234 155 : searchSpecs.put(searchSpec.getName(), searchSpec);
235 155 : return searchSpec;
236 : }
237 :
238 : public SearchSpec exact(String name) {
239 155 : return addSearchSpec(name, SearchOption.EXACT);
240 : }
241 :
242 : public SearchSpec fullText(String name) {
243 155 : return addSearchSpec(name, SearchOption.FULL_TEXT);
244 : }
245 :
246 : public SearchSpec range(String name) {
247 1 : return addSearchSpec(name, SearchOption.RANGE);
248 : }
249 :
250 : public SearchSpec integerRange(String name) {
251 155 : checkState(fieldType().equals(INTEGER_TYPE));
252 155 : return addSearchSpec(name, SearchOption.RANGE);
253 : }
254 :
255 : public SearchSpec integer(String name) {
256 155 : checkState(fieldType().equals(INTEGER_TYPE) || fieldType().equals(ITERABLE_INTEGER_TYPE));
257 155 : return addSearchSpec(name, SearchOption.EXACT);
258 : }
259 :
260 : public SearchSpec longSearch(String name) {
261 1 : checkState(fieldType().equals(LONG_TYPE) || fieldType().equals(ITERABLE_LONG_TYPE));
262 1 : return addSearchSpec(name, SearchOption.EXACT);
263 : }
264 :
265 : public SearchSpec prefix(String name) {
266 155 : return addSearchSpec(name, SearchOption.PREFIX);
267 : }
268 :
269 : public SearchSpec storedOnly(String name) {
270 155 : checkState(stored());
271 155 : return addSearchSpec(name, SearchOption.STORE_ONLY);
272 : }
273 :
274 : public SearchSpec timestamp(String name) {
275 155 : checkState(fieldType().equals(TIMESTAMP_TYPE));
276 155 : return addSearchSpec(name, SearchOption.RANGE);
277 : }
278 :
279 : /** A builder for {@link IndexedField}. */
280 : @AutoValue.Builder
281 155 : public abstract static class Builder<I, T> {
282 :
283 : public abstract IndexedField.Builder<I, T> name(String name);
284 :
285 : public abstract IndexedField.Builder<I, T> description(Optional<String> description);
286 :
287 : public abstract IndexedField.Builder<I, T> description(String description);
288 :
289 : public abstract Builder<I, T> required(boolean required);
290 :
291 : public Builder<I, T> required() {
292 155 : required(true);
293 155 : return this;
294 : }
295 :
296 : /** Allow reading the actual data from the index. */
297 : public abstract Builder<I, T> stored(boolean stored);
298 :
299 : public Builder<I, T> stored() {
300 155 : stored(true);
301 155 : return this;
302 : }
303 :
304 : abstract Builder<I, T> repeatable(boolean repeatable);
305 :
306 : public abstract Builder<I, T> size(Optional<Integer> value);
307 :
308 : public abstract Builder<I, T> size(Integer value);
309 :
310 : public abstract Builder<I, T> getter(Getter<I, T> getter);
311 :
312 : public abstract Builder<I, T> fieldSetter(Optional<Setter<I, T>> setter);
313 :
314 : abstract TypeToken<T> fieldType();
315 :
316 : public abstract Builder<I, T> fieldType(TypeToken<T> type);
317 :
318 : public abstract Builder<I, T> protoConverter(
319 : Optional<ProtoConverter<? extends MessageLite, ?>> value);
320 :
321 : abstract IndexedField<I, T> autoBuild(); // not public
322 :
323 : public final IndexedField<I, T> build() {
324 155 : boolean isRepeatable = fieldType().isSubtypeOf(Iterable.class);
325 155 : repeatable(isRepeatable);
326 155 : IndexedField<I, T> field = autoBuild();
327 155 : checkName(field.name());
328 155 : checkArgument(!field.size().isPresent() || field.size().get() > 0);
329 155 : return field;
330 : }
331 :
332 : public final IndexedField<I, T> build(Getter<I, T> getter, Setter<I, T> setter) {
333 155 : return this.getter(getter).fieldSetter(Optional.of(setter)).build();
334 : }
335 :
336 : public final IndexedField<I, T> build(
337 : Getter<I, T> getter,
338 : Setter<I, T> setter,
339 : ProtoConverter<? extends MessageLite, ?> protoConverter) {
340 1 : return this.getter(getter)
341 1 : .fieldSetter(Optional.of(setter))
342 1 : .protoConverter(Optional.of(protoConverter))
343 1 : .build();
344 : }
345 :
346 : public final IndexedField<I, T> build(Getter<I, T> getter) {
347 155 : return this.getter(getter).fieldSetter(Optional.empty()).build();
348 : }
349 :
350 : private static String checkName(String name) {
351 155 : String allowedCharacters = "abcdefghijklmnopqrstuvwxyz0123456789_";
352 155 : CharMatcher m = CharMatcher.anyOf(allowedCharacters + allowedCharacters.toUpperCase());
353 155 : checkArgument(name != null && m.matchesAllOf(name), "illegal field name: %s", name);
354 155 : return name;
355 : }
356 : }
357 :
358 155 : private Map<String, SearchSpec> searchSpecs = new HashMap<>();
359 :
360 : /**
361 : * The name to store this field under.
362 : *
363 : * <p>The name should use the UpperCamelCase format, see {@link Builder#checkName}.
364 : */
365 : public abstract String name();
366 :
367 : /** Optional description of the field data. */
368 : public abstract Optional<String> description();
369 :
370 : /** True if this field is mandatory. Default is false. */
371 : public abstract boolean required();
372 :
373 : /** Allow reading the actual data from the index. Default is false. */
374 : public abstract boolean stored();
375 :
376 : /** True if this field is repeatable. */
377 : public abstract boolean repeatable();
378 :
379 : /**
380 : * Optional size constrain on the field. The size is not constrained if this property is {@link
381 : * Optional#empty()}
382 : *
383 : * <p>If the field is {@link #repeatable()}, the constraint applies to each element separately.
384 : */
385 : public abstract Optional<Integer> size();
386 :
387 : /** See {@link Getter} */
388 : public abstract Getter<I, T> getter();
389 :
390 : /** See {@link Setter} */
391 : public abstract Optional<Setter<I, T>> fieldSetter();
392 :
393 : /**
394 : * The {@link TypeToken} describing the contents of the field. See static constants for the common
395 : * supported types.
396 : *
397 : * @return {@link TypeToken} of this field.
398 : */
399 : public abstract TypeToken<T> fieldType();
400 :
401 : /** If the {@link #fieldType()} is proto, the converter to use on byte/proto conversions. */
402 : public abstract Optional<ProtoConverter<? extends MessageLite, ?>> protoConverter();
403 :
404 : /**
405 : * Returns all {@link SearchSpec}, enabled on this field.
406 : *
407 : * <p>Note: weather or not a search is supported by the index depends on {@link Schema} version.
408 : */
409 : public ImmutableMap<String, SearchSpec> getSearchSpecs() {
410 153 : return ImmutableMap.copyOf(searchSpecs);
411 : }
412 :
413 : /**
414 : * Get the field contents from the input object.
415 : *
416 : * @param input input object.
417 : * @return the field value(s) to index.
418 : */
419 : @Nullable
420 : public T get(I input) {
421 : try {
422 151 : return getter().get(input);
423 0 : } catch (IOException e) {
424 0 : throw new StorageException(e);
425 : }
426 : }
427 :
428 : @SuppressWarnings("unchecked")
429 : public boolean setIfPossible(I object, StoredValue doc) {
430 101 : if (!fieldSetter().isPresent()) {
431 101 : return false;
432 : }
433 :
434 100 : if (this.fieldType().equals(STRING_TYPE)) {
435 1 : fieldSetter().get().set(object, (T) doc.asString());
436 1 : return true;
437 100 : } else if (this.fieldType().equals(ITERABLE_STRING_TYPE)) {
438 100 : fieldSetter().get().set(object, (T) doc.asStrings());
439 100 : return true;
440 100 : } else if (this.fieldType().equals(INTEGER_TYPE)) {
441 1 : fieldSetter().get().set(object, (T) doc.asInteger());
442 1 : return true;
443 100 : } else if (this.fieldType().equals(ITERABLE_INTEGER_TYPE)) {
444 1 : fieldSetter().get().set(object, (T) doc.asIntegers());
445 1 : return true;
446 100 : } else if (this.fieldType().equals(LONG_TYPE)) {
447 1 : fieldSetter().get().set(object, (T) doc.asLong());
448 1 : return true;
449 100 : } else if (this.fieldType().equals(ITERABLE_LONG_TYPE)) {
450 1 : fieldSetter().get().set(object, (T) doc.asLongs());
451 1 : return true;
452 100 : } else if (this.fieldType().equals(BYTE_ARRAY_TYPE)) {
453 1 : fieldSetter().get().set(object, (T) doc.asByteArray());
454 1 : return true;
455 100 : } else if (this.fieldType().equals(ITERABLE_BYTE_ARRAY_TYPE)) {
456 100 : fieldSetter().get().set(object, (T) doc.asByteArrays());
457 100 : return true;
458 100 : } else if (this.fieldType().equals(TIMESTAMP_TYPE)) {
459 100 : checkState(!repeatable(), "can't repeat timestamp values");
460 100 : fieldSetter().get().set(object, (T) doc.asTimestamp());
461 100 : return true;
462 1 : } else if (isProtoType()) {
463 1 : MessageLite proto = doc.asProto();
464 1 : if (proto != null) {
465 1 : fieldSetter().get().set(object, (T) proto);
466 1 : return true;
467 : }
468 1 : byte[] bytes = doc.asByteArray();
469 1 : if (bytes != null && protoConverter().isPresent()) {
470 1 : fieldSetter().get().set(object, (T) parseProtoFrom(bytes));
471 1 : return true;
472 : }
473 1 : } else if (isProtoIterableType()) {
474 1 : Iterable<MessageLite> protos = doc.asProtos();
475 1 : if (protos != null) {
476 1 : fieldSetter().get().set(object, (T) protos);
477 1 : return true;
478 : }
479 1 : Iterable<byte[]> bytes = doc.asByteArrays();
480 1 : if (bytes != null && protoConverter().isPresent()) {
481 1 : fieldSetter().get().set(object, (T) decodeProtos(bytes));
482 1 : return true;
483 : }
484 : }
485 0 : return false;
486 : }
487 :
488 : /** Returns true if the {@link #fieldType} is a proto message. */
489 : public boolean isProtoType() {
490 1 : if (repeatable()) {
491 1 : return false;
492 : }
493 1 : return MESSAGE_TYPE.isSupertypeOf(fieldType());
494 : }
495 :
496 : /** Returns true if the {@link #fieldType} is a list of proto messages. */
497 : public boolean isProtoIterableType() {
498 1 : if (!repeatable()) {
499 1 : return false;
500 : }
501 1 : if (!(fieldType().getType() instanceof ParameterizedType)) {
502 0 : return false;
503 : }
504 1 : ParameterizedType parameterizedType = (ParameterizedType) fieldType().getType();
505 1 : if (parameterizedType.getActualTypeArguments().length != 1) {
506 0 : return false;
507 : }
508 1 : Type type = parameterizedType.getActualTypeArguments()[0];
509 1 : return MESSAGE_TYPE.isSupertypeOf(type);
510 : }
511 :
512 : private ImmutableList<MessageLite> decodeProtos(Iterable<byte[]> raw) {
513 1 : return StreamSupport.stream(raw.spliterator(), false)
514 1 : .map(bytes -> parseProtoFrom(bytes))
515 1 : .collect(toImmutableList());
516 : }
517 :
518 : private MessageLite parseProtoFrom(byte[] bytes) {
519 1 : return Protos.parseUnchecked(protoConverter().get().getParser(), bytes);
520 : }
521 : }
|
