diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index dcc3f123e97..d5ba2a14dea 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -152,6 +152,7 @@ public final class Constants { public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length"; public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length"; + public static final String INDEX_SEARCH_SOLR_MAX_TOKEN_LENGTH = "atlas.graph.solr.index.search.max-token-length"; public static final String INDEX_SEARCH_VERTEX_PREFIX_PROPERTY = "atlas.graph.index.search.vertex.prefix"; public static final String INDEX_SEARCH_VERTEX_PREFIX_DEFAULT = "$v$"; public static final String MAX_FULLTEXT_QUERY_STR_LENGTH = "atlas.graph.fulltext-max-query-str-length"; diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index ae34f0b82dc..7b5a2455006 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -112,6 +112,7 @@ public abstract class SearchProcessor { public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512); public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512); + public static final int SOLR_MAX_TOKEN_STR_LENGTH = getApplicationProperty(Constants.INDEX_SEARCH_SOLR_MAX_TOKEN_LENGTH, 255); public static final String INDEX_SEARCH_PREFIX = AtlasGraphUtilsV2.getIndexSearchPrefix(); public static final String AND_STR = " AND "; public static final String EMPTY_STRING = ""; @@ -671,6 +672,10 @@ protected AtlasGraphQuery toGraphFilterQuery(Set stru for (AtlasStructType structType : structTypes) { String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName()); + if (isIndexSearchable(criteria, structType)) { + LOG.debug("toGraphFilterQuery() ==> skipped attribute: {}, filter value: {}, and operator: {}", criteria.getAttributeName(), criteria.getAttributeValue(), criteria.getOperator()); + continue; + } if (filterAttributes.contains(qualifiedName)) { String attrName = criteria.getAttributeName(); @@ -866,6 +871,10 @@ private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType } else if (operator == SearchParameters.Operator.CONTAINS && AtlasAttribute.hastokenizeChar(attributeValue) && indexType == null) { // indexType = TEXT LOG.debug("{} operator found for string (TEXT) attribute {} and special characters found in filter value {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName, attributeValue); + ret = false; + } else if ((operator == SearchParameters.Operator.STARTS_WITH || operator == SearchParameters.Operator.ENDS_WITH || operator == SearchParameters.Operator.CONTAINS) && attributeValue.length() > SOLR_MAX_TOKEN_STR_LENGTH) { + LOG.debug("{} operator found for string attribute {} (SOLR MAX TOKEN STR LENGTH:{}) and filter value {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName, SOLR_MAX_TOKEN_STR_LENGTH, attributeValue); + ret = false; } } @@ -920,6 +929,10 @@ private String toIndexQuery(Set structTypes, FilterCr ArrayList orExpQuery = new ArrayList<>(); for (AtlasStructType structType : structTypes) { + if (!isIndexSearchable(criteria, structType)) { + LOG.debug("toIndexQuery() ==> skipped attribute: {}, and filter value: {}, and operator: {}", criteria.getAttributeName(), criteria.getAttributeValue(), criteria.getOperator()); + continue; + } String name = structType.getVertexPropertyName(criteria.getAttributeName()); if (filterAttributes.contains(name)) { diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java index 67e3a45f45f..75957a6502e 100644 --- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java +++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java @@ -92,8 +92,8 @@ public abstract class BasicTestSetup extends AtlasTestBase { protected static final String HIVE_TABLE_TYPE = "hive_table"; protected static final String DATASET_SUBTYPE = "Asset"; protected static final String HDFS_PATH = "hdfs_path"; + protected static final String COLUMN_TYPE = "hive_column"; - private static final String COLUMN_TYPE = "hive_column"; private static final String HIVE_PROCESS_TYPE = "hive_process"; private static final String STORAGE_DESC_TYPE = "hive_storagedesc"; private static final String VIEW_TYPE = "hive_process"; @@ -289,6 +289,45 @@ public AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities() { entities.add(datasetSubType); + List sampleColumns = ImmutableList.of(column("sm_time_id", "int", "time id"), + column("sm_product_id", "int", "product id"), + column("sm_customer_id", "int", "customer id"), + column("sm_sales", "double", "product id"), + column("sm_test", "int", "test 1"), + column("sm_test_limit", "int", "test limit 1")); + + entities.addAll(sampleColumns); + + AtlasEntity sampleTable = table("sample_table", "sample table", salesDB, sd, "Dev 1", "Managed", sampleColumns); + sampleTable.setAttribute("createTime", new Date(2018, 01, 01)); + + entities.add(sampleTable); + + List longTableNameColumns = ImmutableList.of(column("l_id", "int", "time id"), + column("l_product_id", "int", "product id"), + column("l_customer_id", "int", "customer id"), + column("l_sales", "double", "product id"), + column("l_test", "int", "test 1")); + + entities.addAll(longTableNameColumns); + + AtlasEntity longTableName = table("rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb", "table name length 300", salesDB, sd, "Dev 2", "Managed", longTableNameColumns); + longTableName.setAttribute("createTime", new Date(2025, 05, 16)); + + entities.add(longTableName); + + List longTokenizeTableNameColumns = ImmutableList.of(column("l_tknz_id", "int", "time id"), + column("l_tknz_product_id", "int", "product id"), + column("l_tknz_sales", "double", "product id"), + column("l_tknz_test", "int", "test 1")); + + entities.addAll(longTokenizeTableNameColumns); + + AtlasEntity longTokenizeTableName = table("rrrrtokenizeeeeivajaxulwzhgzgzpkfoianndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjogrqzcaqbgpnhxiqbvxcbnudafsmaajzvlzzzzpqmjkngbximmbjbijrq1", "table name length 300, Tokenized", salesDB, sd, "Dev 3", "Managed", longTokenizeTableNameColumns); + longTokenizeTableName.setAttribute("createTime", new Date(2025, 05, 16)); + + entities.add(longTokenizeTableName); + return new AtlasEntity.AtlasEntitiesWithExtInfo(entities); } @@ -317,6 +356,24 @@ public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, return filterCriteria; } + public SearchParameters.FilterCriteria getSingleFilterCriteria(String attName, SearchParameters.Operator op, String attrValue) { + SearchParameters.FilterCriteria f1 = new SearchParameters.FilterCriteria(); + f1.setAttributeName(attName); + f1.setOperator(op); + f1.setAttributeValue(attrValue); + return f1; + } + + public SearchParameters getSearchParameters(String typeName, SearchParameters.FilterCriteria entityFilter, int limit) { + SearchParameters params = new SearchParameters(); + params.setTypeName(typeName); + if (entityFilter != null) { + params.setEntityFilters(entityFilter); + } + params.setLimit(limit); + return params; + } + public void assignGlossary() { try { AtlasGlossary glossary = new AtlasGlossary(); @@ -530,7 +587,7 @@ protected AtlasEntity table(String name, String description, AtlasEntity db, Atl AtlasEntity table = new AtlasEntity(HIVE_TABLE_TYPE); table.setAttribute("name", name); - table.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name); + table.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "@" + clusterName); table.setAttribute("description", description); table.setAttribute("owner", owner); table.setAttribute("tableType", tableType); diff --git a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java index e5cdafa288a..bffe35e9b61 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java @@ -109,7 +109,7 @@ public void term() throws AtlasBaseException { params.setTermName(SALES_TERM + "@" + SALES_GLOSSARY); - assertSearchProcessorWithoutMarker(params, 10); + assertSearchProcessorWithoutMarker(params, 13); } // TSP execute and CSP,ESP filter @@ -130,7 +130,7 @@ public void termEntity() throws AtlasBaseException { params.setTermName(SALES_TERM + "@" + SALES_GLOSSARY); params.setTypeName(HIVE_TABLE_TYPE); - assertSearchProcessorWithoutMarker(params, 10); + assertSearchProcessorWithoutMarker(params, 13); } @Test @@ -458,7 +458,7 @@ public void termMarker() throws AtlasBaseException { params.setTermName(SALES_TERM + "@" + SALES_GLOSSARY); params.setMarker("*"); - assertSearchProcessorWithoutMarker(params, 10); + assertSearchProcessorWithoutMarker(params, 13); } @Test @@ -667,8 +667,8 @@ public void searchWithEntityQuickSearchSortAsc() throws AtlasBaseException { assertTrue(CollectionUtils.isNotEmpty(list)); assertEquals(list.size(), 3); - assertTrue(list.get(0).getAttribute("owner").toString().equalsIgnoreCase("Jane BI")); - assertTrue(list.get(1).getAttribute("owner").toString().equalsIgnoreCase("Joe")); + assertTrue(list.get(0).getAttribute("owner").toString().equalsIgnoreCase("Dev 1")); + assertTrue(list.get(1).getAttribute("owner").toString().equalsIgnoreCase("Dev 2")); } @Test @@ -688,7 +688,7 @@ public void searchWithEntityQuickSearchSortDesc() throws AtlasBaseException { assertTrue(CollectionUtils.isNotEmpty(list)); assertEquals(list.size(), 3); assertTrue(list.get(0).getDisplayText().equalsIgnoreCase("time_dim")); - assertTrue(list.get(1).getDisplayText().equalsIgnoreCase("sales_fact_monthly_mv")); + assertTrue(list.get(1).getDisplayText().equalsIgnoreCase("sample_table")); } @Test diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java index 4e5df065fc3..4bb3cce9cef 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java @@ -199,7 +199,7 @@ public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException { EntitySearchProcessor processor = new EntitySearchProcessor(context); List vertices = processor.execute(); - assertEquals(vertices.size(), 7); + assertEquals(vertices.size(), 10); List nameList = new ArrayList<>(); for (AtlasVertex vertex : vertices) { @@ -277,7 +277,7 @@ public void entityTypes() throws AtlasBaseException { SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.emptySet()); EntitySearchProcessor processor = new EntitySearchProcessor(context); - assertEquals(processor.execute().size(), 14); + assertEquals(processor.execute().size(), 17); } @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename") @@ -376,7 +376,7 @@ public void searchWithNotContains_pipeSeperatedAttr() throws AtlasBaseException EntitySearchProcessor processor = new EntitySearchProcessor(context); List vertices = processor.execute(); - assertEquals(vertices.size(), 7); + assertEquals(vertices.size(), 10); List nameList = new ArrayList<>(); for (AtlasVertex vertex : vertices) { @@ -786,6 +786,131 @@ public void searchWithNameBeginswith() throws AtlasBaseException { assertTrue(guids.contains(cjkGuid2)); } + @Test + public void searchTablesByQualifiedNameBeginswithAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + List lstCriterion = new ArrayList<>(); + + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.STARTS_WITH, "Sales.sample_table")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(HIVE_TABLE_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 1); + } + + @Test + public void searchTablesByLongQualifiedNameBeginswithAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + List lstCriterion = new ArrayList<>(); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.STARTS_WITH, "Sales.rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(HIVE_TABLE_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 1); + } + + @Test + public void searchColumnsByQualifiedNameBeginswithAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + List lstCriterion = new ArrayList<>(); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.STARTS_WITH, "Sales.sample_table.")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(COLUMN_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 6); + } + + @Test + public void searchColumnsByLongQualifiedNameBeginswithAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + List lstCriterion = new ArrayList<>(); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.STARTS_WITH, "Sales.rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb.")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(COLUMN_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 5); + } + + @Test + public void searchTablesByLongTokenizedQualifiedNameBeginswithAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + List lstCriterion = new ArrayList<>(); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.STARTS_WITH, "Sales.rrrrtokenizeeeeivajaxulwzhgzgzpkfoianndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjogrqzcaqbgpnhxiqbvxcbnudafsmaajzvlzzzzpqmjkngbximmbjbijrq1")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(HIVE_TABLE_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 1); + } + + @Test + public void searchTablesByQualifiedNameContainsAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + List lstCriterion = new ArrayList<>(); + + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.CONTAINS, ".sample_table")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(HIVE_TABLE_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 1); + } + + @Test + public void searchTableByLongQualifiedNameContainsAndEndswith() throws AtlasBaseException { + SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); + List lstCriterion = new ArrayList<>(); + + filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND); + + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.CONTAINS, ".rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb")); + lstCriterion.add(getSingleFilterCriteria(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, SearchParameters.Operator.ENDS_WITH, "@cl1")); + filterCriteria.setCriterion(lstCriterion); + + SearchParameters params = getSearchParameters(HIVE_TABLE_TYPE, filterCriteria, 20); + SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys()); + EntitySearchProcessor processor = new EntitySearchProcessor(context); + List vertices = processor.execute(); + + assertEquals(vertices.size(), 1); + } + private static SearchParameters.FilterCriteria filtercriteriaDateRange(String attributeValue, AtlasTypeRegistry typeRegistry, AtlasGraph graph) throws AtlasBaseException { SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria(); SearchParameters params = new SearchParameters(); diff --git a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java index d1f679d5e9d..fac5eb73789 100644 --- a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java +++ b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java @@ -344,18 +344,18 @@ private Object[][] basicQueries() { {"hive_db where hive_db.name=\"Reporting\" select name, owner", 1}, {"hive_column where name='time_id' select name", 1}, {"hive_db has name", 3}, - {"from hive_table", 10}, - {"hive_table", 10}, + {"from hive_table", 13}, + {"hive_table", 13}, {"hive_table isa Dimension", 5}, {"hive_column where hive_column isa PII", 4}, {"hive_column where hive_column isa PII select hive_column.qualifiedName", 4}, - {"hive_column select hive_column.qualifiedName", 21}, - {"hive_column select hive_column.qualifiedName, hive_column.description", 21}, - {"hive_column select qualifiedName", 21}, - {"hive_column select qualifiedName, description", 21}, + {"hive_column select hive_column.qualifiedName", 25}, + {"hive_column select hive_column.qualifiedName, hive_column.description", 25}, + {"hive_column select qualifiedName", 25}, + {"hive_column select qualifiedName, description", 25}, {"hive_column where hive_column.name=\"customer_id\"", 2}, {"hive_column where hive_column.name=\"customer_id\" select qualifiedName, description", 2}, - {"from hive_table select hive_table.qualifiedName", 10}, + {"from hive_table select hive_table.qualifiedName", 13}, {"hive_db where (name = \"Reporting\")", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1}, {"hive_db where hive_db is JdbcAccess", 0}, @@ -404,10 +404,10 @@ private AtlasSearchResult queryAssert(String query, final int expected, final in @DataProvider(name = "limitProvider") private Object[][] limitQueries() { return new Object[][] { - {"hive_column", 21, 40, 0}, + {"hive_column", 36, 40, 0}, {"hive_column limit 10", 10, 50, 0}, {"hive_column select hive_column.qualifiedName limit 10", 10, 5, 0}, - {"hive_column select hive_column.qualifiedName limit 40 offset 10", 11, 40, 0}, + {"hive_column select hive_column.qualifiedName limit 40 offset 10", 26, 40, 0}, {"hive_db where name = 'Reporting' limit 10 offset 0", 1, 40, 0}, {"hive_table where db.name = 'Reporting' limit 10", 4, 1, 0}, {"hive_column where name = 'test' ", 2, DEFAULT_LIMIT, DEFAULT_OFFSET}, @@ -440,11 +440,11 @@ private Object[][] syntaxQueries() { {"hive_db has name limit 10 offset 1", 2}, {"hive_db has name limit 10 offset 0", 3}, - {"from hive_table", 10}, + {"from hive_table", 13}, {"from hive_table limit 5", 5}, {"from hive_table limit 5 offset 5", 5}, - {"hive_table", 10}, + {"hive_table", 13}, {"hive_table limit 5", 5}, {"hive_table limit 5 offset 5", 5}, @@ -453,7 +453,7 @@ private Object[][] syntaxQueries() { {"hive_table isa Dimension limit 2 offset 0", 2}, {"hive_table isa Dimension limit 2 offset 1", 2}, {"hive_table isa Dimension limit 3 offset 1", 3}, - {"hive_table where db.name='Sales' and db.clusterName='cl1'", 4}, + {"hive_table where db.name='Sales' and db.clusterName='cl1'", 7}, {"hive_table where name = 'sales_fact_monthly_mv' and db.name = 'Reporting' and columns.name = 'sales'", 1}, {"hive_column where hive_column isa PII", 4}, @@ -461,11 +461,11 @@ private Object[][] syntaxQueries() { {"hive_column where hive_column isa PII limit 5 offset 1", 3}, {"hive_column where hive_column isa PII limit 5 offset 5", 0}, - {"hive_column select hive_column.qualifiedName", 21}, + {"hive_column select hive_column.qualifiedName", 25}, {"hive_column select hive_column.qualifiedName limit 5", 5}, {"hive_column select hive_column.qualifiedName limit 5 offset 36", 0}, - {"hive_column select qualifiedName", 21}, + {"hive_column select qualifiedName", 25}, {"hive_column select qualifiedName limit 5", 5}, {"hive_column select qualifiedName limit 5 offset 36 ", 0}, @@ -474,7 +474,7 @@ private Object[][] syntaxQueries() { {"hive_column where hive_column.name=\"customer_id\" limit 2 offset 1", 1}, {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 0}, - {"from hive_table select hive_table.name", 10}, + {"from hive_table select hive_table.name", 13}, {"from hive_table select hive_table.name limit 5", 5}, {"from hive_table select hive_table.name limit 5 offset 5", 5}, @@ -505,7 +505,7 @@ private Object[][] syntaxQueries() { {"hive_db as d where owner = ['John ETL', 'Jane BI'] limit 10 offset 1", 1}, {"hive_db where description != '/apps/warehouse/logging'", 2}, {"hive_db where (owner = \"John ETL\" and description != Random)", 1}, - {"hive_table where description != ''", 8}, + {"hive_table where description != ''", 11}, {"hive_db where (name='Reporting' or ((name='Logging' and owner = 'Jane BI') and (name='Logging' and owner = 'John ETL')))", 1} }; } @@ -514,17 +514,17 @@ private Object[][] syntaxQueries() { private Object[][] orderByQueries() { return new Object[][] { {"from hive_db as h orderby h.owner limit 3", 3, "owner", true}, - {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName ", 21, "qualifiedName", true}, + {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName ", 25, "qualifiedName", true}, {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "qualifiedName", true}, {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "qualifiedName", false}, {"from hive_db orderby hive_db.owner limit 3", 3, "owner", true}, - {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 21, "qualifiedName", true}, + {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 25, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "qualifiedName", false}, {"from hive_db orderby owner limit 3", 3, "owner", true}, - {"hive_column select hive_column.qualifiedName orderby qualifiedName ", 21, "qualifiedName", true}, + {"hive_column select hive_column.qualifiedName orderby qualifiedName ", 25, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby qualifiedName limit 5", 5, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby qualifiedName desc limit 5", 5, "qualifiedName", false}, @@ -535,29 +535,29 @@ private Object[][] orderByQueries() { {"hive_db where hive_db.name=\"Reporting\" select name, owner orderby hive_db.name ", 1, "name", true}, {"hive_db has name orderby hive_db.owner limit 10 offset 0", 3, "owner", true}, - {"from hive_table select hive_table.owner orderby hive_table.owner", 10, "owner", true}, + {"from hive_table select hive_table.owner orderby hive_table.owner", 13, "owner", true}, {"from hive_table select hive_table.owner orderby hive_table.owner limit 8", 8, "owner", true}, - {"hive_table orderby hive_table.name", 10, "name", true}, + {"hive_table orderby hive_table.name", 13, "name", true}, - {"hive_table orderby hive_table.owner", 10, "owner", true}, + {"hive_table orderby hive_table.owner", 13, "owner", true}, {"hive_table orderby hive_table.owner limit 8", 8, "owner", true}, {"hive_table orderby hive_table.owner limit 8 offset 0", 8, "owner", true}, {"hive_table orderby hive_table.owner desc limit 8 offset 0", 8, "owner", false}, - {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 21, "qualifiedName", true}, + {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 25, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "qualifiedName", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "qualifiedName", false}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5 offset 2", 5, "qualifiedName", true}, - {"hive_column select qualifiedName orderby hive_column.qualifiedName", 21, "qualifiedName", true}, + {"hive_column select qualifiedName orderby hive_column.qualifiedName", 25, "qualifiedName", true}, {"hive_column select qualifiedName orderby hive_column.qualifiedName limit 5", 5, "qualifiedName", true}, - {"hive_column select qualifiedName orderby hive_column.qualifiedName desc", 21, "qualifiedName", false}, + {"hive_column select qualifiedName orderby hive_column.qualifiedName desc", 25, "qualifiedName", false}, {"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name", 2, "name", true}, {"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name limit 2", 2, "name", true}, {"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name limit 2 offset 1", 1, "name", true}, - {"from hive_table select owner orderby hive_table.owner", 10, "owner", true}, + {"from hive_table select owner orderby hive_table.owner", 13, "owner", true}, {"from hive_table select owner orderby hive_table.owner limit 5", 5, "owner", true}, {"from hive_table select owner orderby hive_table.owner desc limit 5", 5, "owner", false}, {"from hive_table select owner orderby hive_table.owner limit 5 offset 5", 5, "owner", true}, @@ -611,15 +611,15 @@ private Object[][] likeQueries() { return new Object[][] { {"hive_table qualifiedName like \"*time_dim*\"", 1, new ListValidator("time_dim")}, {"hive_db where qualifiedName like \"qualified:R*\"", 1, new ListValidator("Reporting")}, - {"hive_table db.name=\"Sales\"", 4, new ListValidator("customer_dim", "sales_fact", "time_dim", "product_dim")}, - {"hive_table qualifiedName =\"Sales.time_dim\" AND db.name=\"Sales\"", 1, new ListValidator("time_dim")}, + {"hive_table db.name=\"Sales\"", 7, new ListValidator("customer_dim", "sales_fact", "time_dim", "product_dim", "sample_table", "rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb", "rrrrtokenizeeeeivajaxulwzhgzgzpkfoianndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjogrqzcaqbgpnhxiqbvxcbnudafsmaajzvlzzzzpqmjkngbximmbjbijrq1")}, + {"hive_table qualifiedName =\"Sales.time_dim@cl1\" AND db.name=\"Sales\"", 1, new ListValidator("time_dim")}, {"hive_table qualifiedName like \"*time_dim*\" AND db.name=\"Sales\"", 1, new ListValidator("time_dim")}, {"hive_table where name like \"sa?es*\"", 3, new ListValidator("sales_fact", "sales_fact_daily_mv", "sales_fact_monthly_mv")}, {"hive_db where name like \"R*\"", 1, new ListValidator("Reporting")}, {"hive_db where hive_db.name like \"R???rt?*\" or hive_db.name like \"S?l?s\" or hive_db.name like\"Log*\"", 3, new ListValidator("Reporting", "Sales", "Logging")}, {"hive_db where hive_db.name like \"R???rt?*\" and hive_db.name like \"S?l?s\" and hive_db.name like\"Log*\"", 0, new ListValidator()}, {"hive_table where name like 'sales*' and db.name like 'Sa?es'", 1, new ListValidator("sales_fact")}, - {"hive_table where db.name like \"Sa*\"", 4, new ListValidator("customer_dim", "sales_fact", "time_dim", "product_dim")}, + {"hive_table where db.name like \"Sa*\"", 7, new ListValidator("customer_dim", "sales_fact", "time_dim", "product_dim", "sample_table", "rltdvhrxhocivajyqojaxulwzhgzgzpkfolziacfnndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjorziwhogwnjvsdrwksvrbxcxjlcrcmrvvmbdmenafmvgrqzcaqbgpnhxiqbvxcbnudafsmjzvlzzzzpqmjkngbximmbjbijrqfb", "rrrrtokenizeeeeivajaxulwzhgzgzpkfoianndzncjkthzeaeykdhhrjqdeibhdgiepwqkinvqzqxevushydtwjaabzgbfjmzvcbsoxewruhyhciyjefzsnokxvbeiiowzbhlkmujcnwilslgeswobzwwvkugyupsemqxsbdcmgrlpxmeuljvxyddvpccvcloupjogrqzcaqbgpnhxiqbvxcbnudafsmaajzvlzzzzpqmjkngbximmbjbijrq1")}, {"hive_table where db.name like \"Sa*\" and name like \"*dim\"", 3, new ListValidator("customer_dim", "product_dim", "time_dim")}, //STRING Mapping {"hive_db where userDescription like \"*/warehouse/*\"", 3, new ListValidator("Sales", "Reporting", "Logging")}, @@ -722,8 +722,8 @@ private Object[][] minMaxCountQueries() { {"from Asset where __isIncomplete = false groupby (__typeName) select __typeName, count()", new TableValidator("__typeName", "count()") .row("Asset", 1) - .row("hive_table", 10) - .row("hive_column", 21) + .row("hive_table", 13) + .row("hive_column", 36) .row("hive_db", 3) .row("hive_process", 7) }