Skip to content

Improve SQL Server migration process #7292

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
labkey-adam wants to merge 4 commits into
base: develop
Choose a base branch
from
+956 −850
Open

Improve SQL Server migration process #7292

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0477c6d
Improve SQL Server migration process
labkey-adam Dec 24, 2025
6b96cf2
Test auto-increment query
labkey-adam Dec 24, 2025
513d2fb
Move PostgreSQL-specific methods to PostgreSql92Dialect. Fix testLike…
labkey-adam Dec 25, 2025
7f54d35
Eliminate use of "IN (NULL)". Add tests for contains one of, NotClaus…
labkey-adam Dec 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions api/src/org/labkey/api/ApiModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@ public void registerServlets(ServletContext servletCtx)
Aggregate.TestCase.class,
ApiXmlWriter.TestCase.class,
ArrayListMap.TestCase.class,
AssayResultsFileWriter.TestCase.class,
BaseServerProperties.TestCase.class,
BooleanFormat.TestCase.class,
BuilderObjectFactory.TestCase.class,
Expand Down Expand Up @@ -399,12 +400,12 @@ public void registerServlets(ServletContext servletCtx)
GenerateUniqueDataIterator.TestCase.class,
HelpTopic.TestCase.class,
InlineInClauseGenerator.TestCase.class,
JobRunner.TestCase.class,
JSONDataLoader.HeaderMatchTest.class,
JSONDataLoader.MetadataTest.class,
JSONDataLoader.RowTest.class,
JSoupUtil.TestCase.class,
JavaVersion.TestCase.class,
JobRunner.TestCase.class,
JsonTest.class,
JsonUtil.TestCase.class,
LimitedUser.TestCase.class,
Expand All @@ -429,8 +430,10 @@ public void registerServlets(ServletContext servletCtx)
SchemaKey.TestCase.class,
SessionHelper.TestCase.class,
SimpleFilter.BetweenClauseTestCase.class,
SimpleFilter.ContainsOneOfTestCase.class,
SimpleFilter.FilterTestCase.class,
SimpleFilter.InClauseTestCase.class,
SimpleFilter.SqlClauseTestCase.class,
SqlScanner.TestCase.class,
StringExpressionFactory.TestCase.class,
StringUtilsLabKey.TestCase.class,
Expand All @@ -440,9 +443,8 @@ public void registerServlets(ServletContext servletCtx)
TSVWriter.TestCase.class,
TabLoader.HeaderMatchTest.class,
Table.IsSelectTestCase.class,
ValidEmail.TestCase.class,
URIUtil.TestCase.class,
AssayResultsFileWriter.TestCase.class
ValidEmail.TestCase.class
);
}

Expand Down
2 changes: 1 addition & 1 deletion api/src/org/labkey/api/data/CompareType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2108,7 +2108,7 @@ SQLFragment toWhereClause(SqlDialect dialect, DatabaseIdentifier alias)
public String getLabKeySQLWhereClause(Map<FieldKey, ? extends ColumnInfo> columnMap)
{
String colName = getLabKeySQLColName(_fieldKey);
return "LOWER(" + colName + ") LIKE LOWER('%" + escapeLabKeySqlValue(getParamVals()[0], getColumnType(columnMap, JdbcType.VARCHAR), true) + "%') " + sqlEscape();
return "LOWER(" + colName + ") LIKE LOWER('%" + escapeLabKeySqlValue(getParamVals()[0], getColumnType(columnMap, JdbcType.VARCHAR), true) + "%')" + sqlEscape();
}

@Override
Expand Down
11 changes: 10 additions & 1 deletion api/src/org/labkey/api/data/SQLFragment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,12 +313,15 @@ public String toString()
return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this);
}


public String toDebugString()
{
return JdbcUtil.format(this);
}

public String toDebugString(SqlDialect dialect)
{
return JdbcUtil.format(this, dialect);
}

public List<Object> getParams()
{
Expand Down Expand Up @@ -1322,6 +1325,12 @@ public boolean equals(Object obj)
return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams());
}

@Override
public int hashCode()
{
return Objects.hash(getSQL(), getParams());
}

/**
* Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string
* concatenation using the provided separator. The parameters are combined to form the new parameter list.
Expand Down
146 changes: 113 additions & 33 deletions api/src/org/labkey/api/data/SimpleFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,13 +393,27 @@ public String getLabKeySQLWhereClause(Map<FieldKey, ? extends ColumnInfo> column
{
throw new UnsupportedOperationException();
}

@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
SQLClause sqlClause = (SQLClause) o;
return Objects.equals(_fragment, sqlClause._fragment) && Objects.equals(_fieldKeys, sqlClause._fieldKeys);
}

@Override
public int hashCode()
{
return Objects.hash(_fragment, _fieldKeys);
}
}

public static class FalseClause extends SQLClause
{
public FalseClause()
{
super("0=1", null);
super("0 = 1", null);
}
}

Expand Down Expand Up @@ -560,7 +574,7 @@ public Object[] getParamVals()
public SQLFragment toSQLFragment(Map<FieldKey, ? extends ColumnInfo> columnMap, SqlDialect dialect)
{
SQLFragment sqlFragment = new SQLFragment();
sqlFragment.append(" NOT (");
sqlFragment.append("NOT (");
sqlFragment.append(_clause.toSQLFragment(columnMap, dialect));
sqlFragment.append(")");
return sqlFragment;
Expand All @@ -569,14 +583,22 @@ public SQLFragment toSQLFragment(Map<FieldKey, ? extends ColumnInfo> columnMap,
@Override
public String getLabKeySQLWhereClause(Map<FieldKey, ? extends ColumnInfo> columnMap)
{
return " NOT (" + _clause.getLabKeySQLWhereClause(columnMap) + ")";
return "NOT (" + _clause.getLabKeySQLWhereClause(columnMap) + ")";
}

@Override
protected boolean meetsCriteria(ColumnRenderProperties col, Object value)
{
return !_clause.meetsCriteria(col, value);
}

@Override
public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter)
{
sb.append("NOT (");
_clause.appendFilterText(sb, formatter);
sb.append(")");
}
}

public static abstract class MultiValuedFilterClause extends CompareType.AbstractCompareClause
Expand Down Expand Up @@ -726,14 +748,14 @@ public String getLabKeySQLWhereClause(Map<FieldKey, ? extends ColumnInfo> column
String alias = getLabKeySQLColName(getFieldKey());
ColumnInfo col = columnMap != null ? columnMap.get(getFieldKey()) : null;
Object[] params = getParamVals();
StringBuilder in = new StringBuilder();
StringBuilder in = new StringBuilder();

if (params.length == 0)
{
if (isIncludeNull())
in.append(alias).append(" IS ").append(isNegated() ? " NOT " : "").append("NULL");
else if (!isNegated())
in.append(alias).append(" IN (NULL)"); // Empty list case; "WHERE column IN (NULL)" should always be false
in.append(alias).append(" IS ").append(isNegated() ? "NOT " : "").append("NULL");
else
in.append(isNegated() ? "1 = 1" : "0 = 1");
}
else
{
Expand Down Expand Up @@ -780,11 +802,9 @@ else if (!isNegated())
private void handleEmptyParams(DatabaseIdentifier alias, SQLFragment in)
{
if (isIncludeNull())
in.appendIdentifier(alias).append(" IS ").append(isNegated() ? " NOT " : "").append("NULL");
else if (!isNegated())
in.appendIdentifier(alias).append(" IN (NULL)"); // Empty list case; "WHERE column IN (NULL)" should always be false
in.appendIdentifier(alias).append(" IS ").append(isNegated() ? "NOT " : "").append("NULL");
else
in.append("1=1");
in.append(isNegated() ? "1 = 1" : "0 = 1");
}

@Override
Expand Down Expand Up @@ -930,21 +950,21 @@ public String getLabKeySQLWhereClause(Map<FieldKey, ? extends ColumnInfo> column
return getContainsClause(col).getLabKeySQLWhereClause(columnMap);
}

return col.getName() + (isNegated() ? " NOT IN" : " IN ") + " (NULL)"; // Empty list case; "WHERE column IN (NULL)" should always be false
// Empty params list
return isNegated() ? "1 = 1" : "0 = 1";
}

@Override
public SQLFragment toSQLFragment(Map<FieldKey, ? extends ColumnInfo> columnMap, SqlDialect dialect)
{
ColumnInfo colInfo = columnMap != null ? columnMap.get(getFieldKey()) : null;
var alias = getAliasForColumnFilter(dialect, colInfo, getFieldKey());

SQLFragment in = new SQLFragment();
OperationClause oc = getContainsClause(colInfo);
if(!oc.getClauses().isEmpty())
if (!oc.getClauses().isEmpty())
return in.append(oc.toSQLFragment(columnMap, dialect));

return in.appendIdentifier(alias).append(isNegated() ? " NOT IN " : " IN ").append("(NULL)"); // Empty list case; "WHERE column IN (NULL)" should always be false
return in.append(isNegated() ? "1 = 1" : "0 = 1");
}

private OperationClause getContainsClause(ColumnInfo colInfo)
Expand Down Expand Up @@ -1652,9 +1672,10 @@ public void testURLParams() throws URISyntaxException
assertArrayEquals(new Object[] { "x", "u_u" }, containsOneOf.getParamVals());
assertEquals("x;u_u", containsOneOf.toURLParam("query").getValue());
var containsOneOfFrag = containsOneOf.getLabKeySQLWhereClause(Collections.emptyMap());
assertEquals("(LOWER(\"Field2\") LIKE LOWER('%x%') ESCAPE '!')" +
" OR (LOWER(\"Field2\") LIKE LOWER('%u!_u%') ESCAPE '!')",
containsOneOfFrag);
assertEquals(
"(LOWER(\"Field2\") LIKE LOWER('%x%') ESCAPE '!') OR (LOWER(\"Field2\") LIKE LOWER('%u!_u%') ESCAPE '!')",
containsOneOfFrag
);
filter.addClause(containsOneOf);

// verify filter containing multi-value separator ';' use json encoded filter value
Expand All @@ -1672,26 +1693,26 @@ public void testURLParams() throws URISyntaxException
filter.addClause(containsClause);

assertEquals("query.Field1~eq=1" +
"&query.Field2~containsoneof=x%3Bu_u" +
"&query.Field3~containsoneof=" + PageFlowUtil.encodeURIComponent(containsOneOfJsonValue) +
"&query.Field4~contains=o_O",
filter.toQueryString("query"));
"&query.Field2~containsoneof=x%3Bu_u" +
"&query.Field3~containsoneof=" + PageFlowUtil.encodeURIComponent(containsOneOfJsonValue) +
"&query.Field4~contains=o_O",
filter.toQueryString("query"));
URLHelper url = new URLHelper("http://labkey.com");

filter.applyToURL(url, "query");
assertEquals("query.Field1~eq=1" +
"&query.Field2~containsoneof=x%3Bu_u" +
"&query.Field3~containsoneof=" + PageFlowUtil.encodeURIComponent(containsOneOfJsonValue) +
"&query.Field4~contains=o_O",
url.getQueryString());
"&query.Field2~containsoneof=x%3Bu_u" +
"&query.Field3~containsoneof=" + PageFlowUtil.encodeURIComponent(containsOneOfJsonValue) +
"&query.Field4~contains=o_O",
url.getQueryString());
}
}

public abstract static class ClauseTestCase extends Assert
{
protected void test(String expectedSQL, String description, FilterClause clause, SqlDialect dialect, Map<FieldKey, ColumnInfo> columnMap)
{
assertEquals("Generated SQL did not match", expectedSQL, SQLFragment.filterDebugString(clause.toSQLFragment(columnMap, dialect).toDebugString()));
assertEquals("Generated SQL did not match", expectedSQL, SQLFragment.filterDebugString(clause.toSQLFragment(columnMap, dialect).toDebugString(dialect)));
StringBuilder sb = new StringBuilder();
clause.appendFilterText(sb, new ColumnNameFormatter());
assertEquals("Description did not match", description, sb.toString());
Expand All @@ -1701,7 +1722,6 @@ protected void test(String expectedSQL, String description, FilterClause clause,
{
test(expectedSQL, description, clause, dialect, Collections.emptyMap());
}

}

public static class InClauseTestCase extends ClauseTestCase
Expand All @@ -1713,8 +1733,15 @@ public void testInClause()
FieldKey fieldKey = FieldKey.fromParts("Foo");

// Empty parameter list
test("Foo IN (NULL)", "Foo never matches", new InClause(fieldKey, Collections.emptySet()), mockDialect);
test("1=1", "Foo has any value", new InClause(fieldKey, Collections.emptySet(), true, true), mockDialect);
test("0 = 1", "Foo never matches", new InClause(fieldKey, Collections.emptySet()), mockDialect);
test("NOT (0 = 1)", "NOT (Foo never matches)", new NotClause(new InClause(fieldKey, Collections.emptySet())), mockDialect);
test("1 = 1", "Foo has any value", new InClause(fieldKey, Collections.emptySet(), true, true), mockDialect);
test("NOT (1 = 1)", "NOT (Foo has any value)", new NotClause(new InClause(fieldKey, Collections.emptySet(), true, true)), mockDialect);
Map<FieldKey, ColumnInfo> map = Map.of(fieldKey, new BaseColumnInfo(fieldKey, JdbcType.VARCHAR));
assertEquals("0 = 1", new InClause(fieldKey, List.of()).getLabKeySQLWhereClause(map));
assertEquals("NOT (0 = 1)", new NotClause(new InClause(fieldKey, List.of())).getLabKeySQLWhereClause(map));
assertEquals("1 = 1", new InClause(fieldKey, List.of(), false, true).getLabKeySQLWhereClause(map));
assertEquals("NOT (1 = 1)", new NotClause(new InClause(fieldKey, List.of(), false, true)).getLabKeySQLWhereClause(map));

// Non-null parameters only
test("(Foo IN (1, 2, 3))", "Foo IS ONE OF (1, 2, 3)", new InClause(fieldKey, PageFlowUtil.set(1, 2, 3)), mockDialect);
Expand All @@ -1737,13 +1764,15 @@ public void testInClause()
in._needsTypeConversion = true;
test("((NOT \"FOO\" IN (1, 2)) OR \"FOO\" IS NULL)", "Foo IS NOT ANY OF (1, 2, S-3)", in, mockDialect, columnInfoMap);

in = new InClause(fieldKey, PageFlowUtil.set("S-3"));
in = new InClause(fieldKey, PageFlowUtil.set("S-3"));
in._needsTypeConversion = true;
test("\"FOO\" IN (NULL)", "Foo IS ONE OF (S-3)", in, mockDialect, columnInfoMap);
test("0 = 1", "Foo IS ONE OF (S-3)", in, mockDialect, columnInfoMap);
test("NOT (0 = 1)", "NOT (Foo IS ONE OF (S-3))", new NotClause(in), mockDialect, columnInfoMap);

in = new InClause(fieldKey, PageFlowUtil.set("S-3"), true, true);
in._needsTypeConversion = true;
test("1=1", "Foo IS NOT ANY OF (S-3)", in, mockDialect, columnInfoMap);
test("1 = 1", "Foo IS NOT ANY OF (S-3)", in, mockDialect, columnInfoMap);
test("NOT (1 = 1)", "NOT (Foo IS NOT ANY OF (S-3))", new NotClause(in), mockDialect, columnInfoMap);
}

@Test
Expand Down Expand Up @@ -1953,4 +1982,55 @@ public void testBetweenQueryString()
new CompareType.BetweenClause(fieldKey, "a,b,c", "Z", false).toURLParam("query."));
}
}

public static class SqlClauseTestCase extends Assert
{
@Test
public void testEqualsAndHashCode()
{
SQLClause clause1 = new SQLClause(new SQLFragment("This = That", 1, 2));
SQLClause clause2 = new SQLClause(new SQLFragment("This = That", 1, 2));
assertEquals(clause1, clause2);
assertEquals(clause1.hashCode(), clause2.hashCode());
SQLClause clause3 = new SQLClause(new SQLFragment("That = This", 1, 2));
assertNotEquals(clause1, clause3);
assertNotEquals(clause1.hashCode(), clause3.hashCode());
SQLClause clause4 = new SQLClause(new SQLFragment("This = That", 3, 4));
assertNotEquals(clause1, clause4);
assertNotEquals(clause1.hashCode(), clause4.hashCode());
}
}

public static class ContainsOneOfTestCase extends ClauseTestCase
{
@Test
public void testContainsOneOf()
{
SqlDialect dialect = DbScope.getLabKeyScope().getSqlDialect();
FieldKey fieldKey = FieldKey.fromParts("Foo");
String like = dialect.getCaseInsensitiveLikeOperator();
String concat = dialect.isPostgreSQL() ? "||" : "+";

FilterClause clause = new ContainsOneOfClause(fieldKey, List.of("This", "That"), false);
test("(Foo " + like + " '%' " + concat + " 'This' " + concat + " '%' ESCAPE '!') OR (Foo " + like + " '%' " + concat + " 'That' " + concat + " '%' ESCAPE '!')", "Foo CONTAINS ONE OF (This, That)", clause, dialect);
test("NOT ((Foo " + like + " '%' " + concat + " 'This' " + concat + " '%' ESCAPE '!') OR (Foo " + like + " '%' " + concat + " 'That' " + concat + " '%' ESCAPE '!'))", "NOT (Foo CONTAINS ONE OF (This, That))", new NotClause(clause), dialect);

clause = new ContainsOneOfClause(fieldKey, List.of("This", "That"), false, true);
test("((Foo IS NULL OR Foo NOT " + like + " '%' " + concat + " 'This' " + concat + " '%' ESCAPE '!')) AND ((Foo IS NULL OR Foo NOT " + like + " '%' " + concat + " 'That' " + concat + " '%' ESCAPE '!'))", "Foo DOES NOT CONTAIN ANY OF (This, That)", clause, dialect);
test("NOT (((Foo IS NULL OR Foo NOT " + like + " '%' " + concat + " 'This' " + concat + " '%' ESCAPE '!')) AND ((Foo IS NULL OR Foo NOT " + like + " '%' " + concat + " 'That' " + concat + " '%' ESCAPE '!')))", "NOT (Foo DOES NOT CONTAIN ANY OF (This, That))", new NotClause(clause), dialect);

Map<FieldKey, ColumnInfo> map = Map.of(fieldKey, new BaseColumnInfo(fieldKey, JdbcType.VARCHAR));
clause = new ContainsOneOfClause(fieldKey, List.of(), false);
test("0 = 1", "Foo CONTAINS ONE OF (BLANK)", clause, dialect);
test("NOT (0 = 1)", "NOT (Foo CONTAINS ONE OF (BLANK))", new NotClause(clause), dialect);
assertEquals("0 = 1", clause.getLabKeySQLWhereClause(map));
assertEquals("NOT (0 = 1)", new NotClause(clause).getLabKeySQLWhereClause(map));

clause = new ContainsOneOfClause(fieldKey, List.of(), false, true);
test("1 = 1", "Foo DOES NOT CONTAIN ANY OF (BLANK)", clause, dialect);
test("NOT (1 = 1)", "NOT (Foo DOES NOT CONTAIN ANY OF (BLANK))", new NotClause(clause), dialect);
assertEquals("1 = 1", clause.getLabKeySQLWhereClause(map));
assertEquals("NOT (1 = 1)", new NotClause(clause).getLabKeySQLWhereClause(map));
}
}
}
Loading