Merge remote-tracking branch 'origin/banyandb' into banyandb

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java

@@ -275,6 +275,10 @@ public class OALRuntime implements OALEngine {
                     banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0));
                     annotationsAttribute.addAnnotation(banyanShardingKeyAnnotation);
                 }
+                if (field.isBanyandbNoIndexing()) {
+                    Annotation banyandbNoIndexingAnnotation = new Annotation(BanyanDB.NoIndexing.class.getName(), constPool);
+                    annotationsAttribute.addAnnotation(banyandbNoIndexingAnnotation);
+                }
 
                 newField.getFieldInfo().addAttribute(annotationsAttribute);
             } catch (CannotCompileException e) {

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java

@@ -34,14 +34,16 @@ public class SourceColumn {
     private int length;
     private String fieldSetter;
     private String fieldGetter;
+    private boolean banyandbNoIndexing;
 
-    public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
+    public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length, boolean banyandbNoIndexing) {
         this.fieldName = fieldName;
         this.columnName = columnName;
         this.type = type;
         this.typeName = type.getName();
         this.isID = isID;
         this.length = length;
+        this.banyandbNoIndexing = banyandbNoIndexing;
 
         this.fieldGetter = ClassMethodUtil.toGetMethod(fieldName);
         this.fieldSetter = ClassMethodUtil.toSetMethod(fieldName);
@@ -79,7 +81,7 @@ public class SourceColumn {
 
     @Override
     public String toString() {
-        return "SourceColumn{" + "fieldName='" + fieldName + '\'' + ", columnName='" + columnName + '\'' + ", type=" + type + ", isID=" + isID + '}';
+        return "SourceColumn{" + "fieldName='" + fieldName + '\'' + ", columnName='" + columnName + '\'' + ", type=" + type + ", isID=" + isID + ", banyandbNoIndexing=" + banyandbNoIndexing + "}";
     }
 
     @Override
@@ -89,13 +91,13 @@ public class SourceColumn {
         if (o == null || getClass() != o.getClass())
             return false;
         SourceColumn column = (SourceColumn) o;
-        return isID == column.isID && Objects.equals(fieldName, column.fieldName) && Objects.equals(columnName, column.columnName) && Objects
-            .equals(type, column.type) && Objects.equals(typeName, column.typeName) && Objects.equals(fieldSetter, column.fieldSetter) && Objects
-            .equals(fieldGetter, column.fieldGetter);
+        return isID == column.isID && banyandbNoIndexing == column.isBanyandbNoIndexing() && Objects.equals(fieldName, column.fieldName) && Objects
+                .equals(columnName, column.columnName) && Objects.equals(type, column.type) && Objects.equals(typeName, column.typeName) && Objects
+                .equals(fieldSetter, column.fieldSetter) && Objects.equals(fieldGetter, column.fieldGetter);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fieldName, columnName, type, typeName, isID, fieldSetter, fieldGetter);
+        return Objects.hash(fieldName, columnName, type, typeName, isID, fieldSetter, fieldGetter, banyandbNoIndexing);
     }
 }

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java

@@ -31,7 +31,7 @@ public class SourceColumnsFactory {
         for (ScopeDefaultColumn defaultColumn : columns) {
             sourceColumns.add(
                 new SourceColumn(defaultColumn.getFieldName(), defaultColumn.getColumnName(), defaultColumn
-                    .getType(), defaultColumn.isID(), defaultColumn.getLength()));
+                    .getType(), defaultColumn.isID(), defaultColumn.getLength(), defaultColumn.isBanyandbNoIndexing()));
         }
         return sourceColumns;
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java

@@ -61,6 +61,7 @@ public abstract class AvgFunction extends Meter implements AcceptableValue<Long>
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java

@@ -60,6 +60,7 @@ public abstract class AvgLabeledFunction extends Meter implements AcceptableValu
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java

@@ -58,6 +58,7 @@ public abstract class LatestFunction extends Meter implements AcceptableValue<Lo
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java

@@ -55,6 +55,7 @@ public abstract class SumFunction extends Meter implements AcceptableValue<Long>
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sumpermin/SumPerMinFunction.java

@@ -56,6 +56,7 @@ public abstract class SumPerMinFunction extends Meter implements AcceptableValue
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sumpermin/SumPerMinLabeledFunction.java

@@ -55,6 +55,7 @@ public abstract class SumPerMinLabeledFunction extends Meter implements Acceptab
     @Setter
     @Getter
     @Column(columnName = InstanceTraffic.SERVICE_ID)
+    @BanyanDB.NoIndexing
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java

@@ -203,7 +203,7 @@ public class DefaultScopeDefine {
         if (virtualColumn != null) {
             scopeDefaultColumns.add(
                 new ScopeDefaultColumn(virtualColumn.fieldName(), virtualColumn.columnName(), virtualColumn
-                    .type(), virtualColumn.isID(), virtualColumn.length()));
+                    .type(), virtualColumn.isID(), virtualColumn.length(), false));
         }
         Field[] scopeClassField = originalClass.getDeclaredFields();
         if (scopeClassField != null) {
@@ -215,7 +215,7 @@ public class DefaultScopeDefine {
                         scopeDefaultColumns.add(
                             new ScopeDefaultColumn(
                                 field.getName(), definedByField.columnName(), field.getType(), false,
-                                definedByField.length()
+                                definedByField.length(), definedByField.requiredNoIndexingForBanyanDB()
                             ));
                     }
                 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java

@@ -54,7 +54,7 @@ public class Endpoint extends Source {
     @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/MQEndpointAccess.java

@@ -43,7 +43,7 @@ public class MQEndpointAccess extends Source {
     private String entityId;
 
     @Getter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
 
     @Getter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java

@@ -34,14 +34,16 @@ public class ScopeDefaultColumn {
     private String columnName;
     private Class<?> type;
     private boolean isID;
+    private boolean banyandbNoIndexing;
     private int length;
 
-    public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
+    public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length, boolean banyandbNoIndexing) {
         this.fieldName = fieldName;
         this.columnName = columnName;
         this.type = type;
         this.isID = isID;
         this.length = length;
+        this.banyandbNoIndexing = banyandbNoIndexing;
     }
 
     @Target({ElementType.FIELD})
@@ -52,7 +54,7 @@ public class ScopeDefaultColumn {
         /**
          * Dynamic active means this column is only activated through core setting explicitly.
          *
-         * @return
+         * @return false: Inactive dynamic column. true: Active dynamic column.
          */
         boolean requireDynamicActive() default false;
 
@@ -60,6 +62,14 @@ public class ScopeDefaultColumn {
          * Define column length, only effective when the type is String.
          */
         int length() default 256;
+
+
+        /**
+         * Active indexing for a column(tag) in BanyanDB.
+         *
+         * @return false: Indexing the column, true: No indexing the column.
+         */
+        boolean requiredNoIndexingForBanyanDB() default false;
     }
 
     @Target({ElementType.TYPE})

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java

@@ -47,7 +47,7 @@ public class ServiceInstance extends Source {
     }
 
     @Getter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java

@@ -53,7 +53,7 @@ public class ServiceInstanceCLRCPU extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java

@@ -53,7 +53,7 @@ public class ServiceInstanceCLRGC extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java

@@ -53,7 +53,7 @@ public class ServiceInstanceCLRThread extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMCPU extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMClass.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMClass extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMGC extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMMemory extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMMemoryPool extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMThread.java

@@ -50,7 +50,7 @@ public class ServiceInstanceJVMThread extends Source {
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_id", requiredNoIndexingForBanyanDB = true)
     private String serviceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java

@@ -104,6 +104,9 @@ public class ModelColumn {
      * @return true means this column should be indexed, as it would be a query condition.
      */
     public boolean shouldIndex() {
+        if (banyanDBExtension != null) {
+            return banyanDBExtension.shouldIndex();
+        }
         return !storageOnly;
     }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java

@@ -211,7 +211,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
                 BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
                     banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(),
                     banyanDBGlobalIndex != null,
-                    banyanDBNoIndex == null && column.storageOnly(),
+                    banyanDBNoIndex == null && !column.storageOnly(),
                     banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(),
                         banyanDBMeasureField != null
                 );

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java

@@ -89,11 +89,14 @@ public enum MetadataRegistry {
                 .collect(Collectors.toMap(modelColumn -> modelColumn.getColumnName().getStorageName(), Function.identity()));
         // parse and set sharding keys
         List<String> shardingColumns = parseEntityNames(modelColumnMap);
+        if (shardingColumns.isEmpty()) {
+            throw new IllegalStateException("sharding keys of model[stream." + model.getName() + "] must not be empty");
+        }
         // parse tag metadata
         // this can be used to build both
         // 1) a list of TagFamilySpec,
         // 2) a list of IndexRule,
-        List<TagMetadata> tags = parseTagMetadata(model, schemaBuilder);
+        List<TagMetadata> tags = parseTagMetadata(model, schemaBuilder, shardingColumns);
         List<TagFamilySpec> tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tags, false);
         // iterate over tagFamilySpecs to save tag names
         for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) {
@@ -113,9 +116,6 @@ public enum MetadataRegistry {
                 .collect(Collectors.toList());
 
         final Stream.Builder builder = Stream.create(schemaMetadata.getGroup(), schemaMetadata.name());
-        if (shardingColumns.isEmpty()) {
-            throw new IllegalStateException("sharding keys of model[stream." + model.getName() + "] must not be empty");
-        }
         builder.setEntityRelativeTags(shardingColumns);
         builder.addTagFamilies(tagFamilySpecs);
         builder.addIndexes(indexRules);
@@ -137,7 +137,7 @@ public enum MetadataRegistry {
         // this can be used to build both
         // 1) a list of TagFamilySpec,
         // 2) a list of IndexRule,
-        MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder);
+        MeasureMetadata tagsAndFields = parseTagAndFieldMetadata(model, schemaBuilder, shardingColumns);
         List<TagFamilySpec> tagFamilySpecs = schemaMetadata.extractTagFamilySpec(tagsAndFields.tags, model.getBanyanDBModelExtension().isShouldStoreIDTag());
         // iterate over tagFamilySpecs to save tag names
         for (final TagFamilySpec tagFamilySpec : tagFamilySpecs) {
@@ -158,7 +158,9 @@ public enum MetadataRegistry {
                 downSamplingDuration(model.getDownsampling()));
         builder.setEntityRelativeTags(shardingColumns);
         builder.addTagFamilies(tagFamilySpecs);
-        builder.addIndexes(indexRules);
+        if (!indexRules.isEmpty()) {
+            builder.addIndexes(indexRules);
+        }
         // parse and set field
         for (Measure.FieldSpec field : tagsAndFields.fields) {
             builder.addField(field);
@@ -297,7 +299,7 @@ public enum MetadataRegistry {
      *
      * @since 9.4.0 Skip {@link Record#TIME_BUCKET}
      */
-    List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder) {
+    List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder, List<String> shardingColumns) {
         List<TagMetadata> tagMetadataList = new ArrayList<>();
         for (final ModelColumn col : model.getColumns()) {
             final String columnStorageName = col.getColumnName().getStorageName();
@@ -306,7 +308,8 @@ public enum MetadataRegistry {
             }
             final TagFamilySpec.TagSpec tagSpec = parseTagSpec(col);
             builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
-            if (col.shouldIndex()) {
+            String colName = col.getColumnName().getStorageName();
+            if (!shardingColumns.contains(colName) && col.shouldIndex()) {
                 // build indexRule
                 IndexRule indexRule = parseIndexRule(tagSpec.getTagName(), col);
                 tagMetadataList.add(new TagMetadata(indexRule, tagSpec));
@@ -333,7 +336,7 @@ public enum MetadataRegistry {
      *
      * @since 9.4.0 Skip {@link Metrics#TIME_BUCKET}
      */
-    MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder) {
+    MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder, List<String> shardingColumns) {
         // skip metric
         Optional<ValueColumnMetadata.ValueColumn> valueColumnOpt = ValueColumnMetadata.INSTANCE
                 .readValueColumnDefinition(model.getName());
@@ -350,7 +353,8 @@ public enum MetadataRegistry {
             }
             final TagFamilySpec.TagSpec tagSpec = parseTagSpec(col);
             builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
-            result.tag(new TagMetadata(col.shouldIndex() ? parseIndexRule(tagSpec.getTagName(), col) : null, tagSpec));
+            String colName = col.getColumnName().getStorageName();
+            result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.shouldIndex() ? parseIndexRule(tagSpec.getTagName(), col) : null, tagSpec));
         }
 
         return result.build();