From d50296ca8879c9b5597f1314bdaf13632b88ce90 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 21 Apr 2026 14:29:55 -0700 Subject: [PATCH 01/10] minimal test (thanks Claude) --- query/src/org/labkey/query/QueryModule.java | 3 +- .../query/controllers/SqlController.java | 171 ++++++++++++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 15301c414c9..54e7760c86e 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -397,7 +397,8 @@ public Set getSchemaNames() RolapReader.RolapTest.class, RolapTestCase.class, SelectRowsStreamHack.TestCase.class, - ServerManager.TestCase.class + ServerManager.TestCase.class, + SqlController.TestCase.class ); } diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index 3a9b64b6a33..d7dcf18d236 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -17,13 +17,25 @@ import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.commons.lang3.StringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; import org.labkey.api.action.Marshal; import org.labkey.api.action.Marshaller; import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.Results; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.QueryParseException; import org.labkey.api.query.QuerySchema; @@ -31,18 +43,26 @@ import org.labkey.api.query.SchemaKey; import org.labkey.api.query.UserSchema; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.User; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.util.DateUtil; +import org.labkey.api.util.JunitUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewServlet; import org.springframework.beans.PropertyValue; +import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.validation.BindException; import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.math.BigDecimal; import java.sql.SQLException; import java.util.Date; +import java.util.List; import java.util.Map; /** @@ -442,4 +462,155 @@ else if (serializeDateAsNumber) } out.flush(); } + + + public static class TestCase extends Assert + { + private static final String FOLDER_NAME = "sqlControllerTest"; + private static final String LIST_NAME = "SqlTestList"; + + private Container _folder; + + @Before + public void setUp() throws Exception + { + tearDown(); + Assume.assumeTrue("Requires list module", ListService.get() != null); + + User user = TestContext.get().getUser(); + _folder = ContainerManager.ensureContainer(JunitUtil.getTestContainer().getPath() + "/" + FOLDER_NAME, user); + + ListDefinition list = ListService.get().createList(_folder, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); + list.setKeyName("Key"); + list.getDomain().addProperty(new PropertyStorageSpec("Name", JdbcType.VARCHAR)); + list.getDomain().addProperty(new PropertyStorageSpec("Age", JdbcType.INTEGER)); + list.getDomain().addProperty(new PropertyStorageSpec("Score", JdbcType.DOUBLE)); + list.save(user); + + TableInfo table = DefaultSchema.get(user, _folder).getSchema("lists").getTable(LIST_NAME, null); + assertNotNull("List table not found", table); + + BatchValidationException errors = new BatchValidationException(); + table.getUpdateService().insertRows(user, _folder, List.of( + CaseInsensitiveHashMap.of("Name", "Alice", "Age", 30, "Score", 95.5), + CaseInsensitiveHashMap.of("Name", "Bob", "Age", 30, "Score", 87.3), + CaseInsensitiveHashMap.of("Name", "Carol", "Age", 35, "Score", 91.0) + ), errors, null, null); + if (errors.hasErrors()) + fail(errors.getRowErrors().get(0).toString()); + } + + @After + public void tearDown() + { + Container folder = ContainerManager.getForPath(JunitUtil.getTestContainer().getPath() + "/" + FOLDER_NAME); + if (folder != null) + ContainerManager.deleteAll(folder, TestContext.get().getUser()); + _folder = null; + } + + private MockHttpServletResponse executeSql(String schemaName, String sql, boolean compact) throws Exception + { + ActionURL url = new ActionURL("sql", "execute", _folder); + if (schemaName != null) + url.addParameter("schemaName", schemaName); + if (sql != null) + url.addParameter("sql", sql); + if (compact) + url.addParameter("compact", true); + return ViewServlet.GET(url, TestContext.get().getUser(), null); + } + + @Test + public void testExecute() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", false); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] tokens = content.split("\t"); + + // Header: meta-meta-data (3) + column names (3) + types (3) = 9 + // Data: 3 rows * 3 columns = 9 + assertTrue("Expected at least 18 tokens, got " + tokens.length, tokens.length >= 18); + + // Meta-meta-data + assertEquals("18.2", tokens[0]); + assertEquals("name", tokens[1]); + assertEquals("jdbcType", tokens[2]); + + // Column names + assertEquals("Name", tokens[3]); + assertEquals("Age", tokens[4]); + assertEquals("Score", tokens[5]); + + // JDBC types + assertEquals("VARCHAR", tokens[6]); + assertEquals("INTEGER", tokens[7]); + assertEquals("DOUBLE", tokens[8]); + + // Data rows ordered by Name + assertEquals("Alice", tokens[9]); + assertEquals("30", tokens[10]); + assertEquals("Bob", tokens[12]); + assertEquals("30", tokens[13]); + assertEquals("Carol", tokens[15]); + assertEquals("35", tokens[16]); + } + + @Test + public void testExecuteCompact() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", true); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String sep = "\u001f"; + String eol = "\u001e"; + String ditto = "\u0008"; + + String[] records = content.split(eol); + // records: [0]=meta, [1]=column names, [2]=types, [3..5]=data rows + assertTrue("Expected at least 6 records", records.length >= 6); + + // Column names + String[] colNames = records[1].split(sep); + assertEquals("Name", colNames[0]); + assertEquals("Age", colNames[1]); + + // First data row: Alice, 30 + String[] row1 = records[3].split(sep, -1); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + + // Second data row: Bob, 30 (ditto marker since Age repeats) + String[] row2 = records[4].split(sep, -1); + assertEquals("Bob", row2[0]); + assertEquals(ditto, row2[1]); + + // Third data row: Carol, 35 + String[] row3 = records[5].split(sep, -1); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + } + + @Test + public void testNoSql() throws Exception + { + MockHttpServletResponse response = executeSql("lists", null, false); + assertTrue("Expected error about missing SQL", + response.getContentAsString().contains("no sql provided")); + } + + @Test + public void testSchemaNotFound() throws Exception + { + MockHttpServletResponse response = executeSql("nonexistent", + "SELECT 1", false); + assertTrue("Expected schema not found error", + response.getContentAsString().contains("schema not found")); + } + } } \ No newline at end of file From c94620a3cb85f6c97a4d414c8b0c7a89421a4ef9 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 21 Apr 2026 15:37:48 -0700 Subject: [PATCH 02/10] MVTC --- .../query/controllers/SqlController.java | 74 +++++++++++++++++-- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index d7dcf18d236..93326ecdcfc 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -29,12 +29,17 @@ import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.JdbcType; import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.Results; import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.list.ListDefinition; import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.PropertyService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.QueryParseException; @@ -485,6 +490,17 @@ public void setUp() throws Exception list.getDomain().addProperty(new PropertyStorageSpec("Name", JdbcType.VARCHAR)); list.getDomain().addProperty(new PropertyStorageSpec("Age", JdbcType.INTEGER)); list.getDomain().addProperty(new PropertyStorageSpec("Score", JdbcType.DOUBLE)); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + DomainProperty tagsProp = list.getDomain().addProperty(new PropertyStorageSpec("Tags", JdbcType.VARCHAR)); + tagsProp.setRangeURI(PropertyType.MULTI_CHOICE.getTypeUri()); + IPropertyValidator tcValidator = PropertyService.get().createValidator("urn:lsid:labkey.com:PropertyValidator:textchoice"); + tcValidator.setName("Text Choice Validator"); + tcValidator.setExpressionValue("Red|Green|Blue"); + tagsProp.addValidator(tcValidator); + } + list.save(user); TableInfo table = DefaultSchema.get(user, _folder).getSchema("lists").getTable(LIST_NAME, null); @@ -492,9 +508,9 @@ public void setUp() throws Exception BatchValidationException errors = new BatchValidationException(); table.getUpdateService().insertRows(user, _folder, List.of( - CaseInsensitiveHashMap.of("Name", "Alice", "Age", 30, "Score", 95.5), - CaseInsensitiveHashMap.of("Name", "Bob", "Age", 30, "Score", 87.3), - CaseInsensitiveHashMap.of("Name", "Carol", "Age", 35, "Score", 91.0) + CaseInsensitiveHashMap.of("Name", "Alice", "Age", 30, "Score", 95.5, "Tags", List.of("Red", "Green")), + CaseInsensitiveHashMap.of("Name", "Bob", "Age", 30, "Score", 87.3, "Tags", List.of("Blue")), + CaseInsensitiveHashMap.of("Name", "Carol", "Age", 35, "Score", 91.0, "Tags", List.of("Red", "Blue", "Green")) ), errors, null, null); if (errors.hasErrors()) fail(errors.getRowErrors().get(0).toString()); @@ -522,10 +538,10 @@ private MockHttpServletResponse executeSql(String schemaName, String sql, boolea } @Test - public void testExecute() throws Exception + public void testExecute_mssql() throws Exception { MockHttpServletResponse response = executeSql("lists", - "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", false); + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", false); assertEquals(HttpServletResponse.SC_OK, response.getStatus()); String content = response.getContentAsString(); @@ -558,6 +574,54 @@ public void testExecute() throws Exception assertEquals("Carol", tokens[15]); assertEquals("35", tokens[16]); } + @Test + public void testExecute() throws Exception + { + if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + testExecute_mssql(); + return; + } + + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score, Tags FROM " + LIST_NAME + " ORDER BY Name", false); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] tokens = content.split("\t"); + + // Header: meta-meta-data (3) + column names (4) + types (4) = 11 + // Data: 3 rows * 4 columns = 12 + assertTrue("Expected at least 23 tokens, got " + tokens.length, tokens.length >= 23); + + // Meta-meta-data + assertEquals("18.2", tokens[0]); + assertEquals("name", tokens[1]); + assertEquals("jdbcType", tokens[2]); + + // Column names + assertEquals("Name", tokens[3]); + assertEquals("Age", tokens[4]); + assertEquals("Score", tokens[5]); + assertEquals("Tags", tokens[6]); + + // JDBC types + assertEquals("VARCHAR", tokens[7]); + assertEquals("INTEGER", tokens[8]); + assertEquals("DOUBLE", tokens[9]); + assertEquals("ARRAY", tokens[10]); + + // Data rows ordered by Name (4 columns per row) + assertEquals("Alice", tokens[11]); + assertEquals("30", tokens[12]); + assertTrue("Alice Tags", tokens[14].contains("Red") && tokens[14].contains("Green")); + assertEquals("Bob", tokens[15]); + assertEquals("30", tokens[16]); + assertTrue("Bob Tags", tokens[18].contains("Blue")); + assertEquals("Carol", tokens[19]); + assertEquals("35", tokens[20]); + assertTrue("Carol Tags", tokens[22].contains("Red") && tokens[22].contains("Blue") && tokens[22].contains("Green")); + } @Test public void testExecuteCompact() throws Exception From ab28bc53a586cd5b7c99f06c86244edc7bb35fa9 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 23 Apr 2026 13:18:56 -0700 Subject: [PATCH 03/10] sql-execute refactor tsv option (LLM friendly) --- api/src/org/labkey/api/util/PageFlowUtil.java | 10 + .../query/controllers/SqlController.java | 424 +++++++++++------- 2 files changed, 266 insertions(+), 168 deletions(-) diff --git a/api/src/org/labkey/api/util/PageFlowUtil.java b/api/src/org/labkey/api/util/PageFlowUtil.java index eb2ff42bb80..bbfbabda72c 100644 --- a/api/src/org/labkey/api/util/PageFlowUtil.java +++ b/api/src/org/labkey/api/util/PageFlowUtil.java @@ -2676,6 +2676,16 @@ private static boolean shouldEscapeForExport(@NotNull String value) return StringUtils.containsAny(value,",\""); } + /// Generate one row of tab-delimited output using RFC 4180 quoting rules. + /// Fields containing tabs, newlines, or double quotes are enclosed in double quotes, + /// with embedded double quotes escaped by doubling. + public static String joinValuesWithTabs4180(@NotNull List values) + { + return values.stream() + .map(value -> null == value ? "" : StringUtils.containsAny(value, "\t\n\r\"") ? "\"" + Strings.CS.replace(value, "\"", "\"\"") + "\"" : value) + .collect(Collectors.joining("\t")); + } + /** * Issue 52925: App export to csv/tsv ignores filter with column containing double quote diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index 93326ecdcfc..82d9c13ae20 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -16,6 +16,8 @@ package org.labkey.query.controllers; import com.fasterxml.jackson.annotation.JsonAnySetter; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Assert; @@ -27,8 +29,10 @@ import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.JdbcType; import org.labkey.api.data.PropertyStorageSpec; @@ -40,6 +44,7 @@ import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.IPropertyValidator; import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.QueryParseException; @@ -50,7 +55,6 @@ import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.User; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.util.DateUtil; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.TestContext; @@ -60,13 +64,12 @@ import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.validation.BindException; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.math.BigDecimal; +import java.sql.Array; import java.sql.SQLException; -import java.util.Date; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -94,14 +97,30 @@ public void set(String name, Object value) } } + public enum Format + { + split, // respose that can be parsed using String.split(), configure with 'sep' and 'eol' + + compact, // same as split, but with ditto markers (cheap way to compress and save space on client) + + // Tab separated (RFC 4180) + // - Wrap a field in double quotes if it contains a tab, newline, or double quote + // - Escape double quotes by doubling them: " becomes "" + // - Fields that don't contain special characters are left unquoted + tsv + + // We could support CSV here, I can't think of a scenario for this API where this would be preferable + // csv // Comma separated, google sheets style quoting (see PageFlowUtil.joinValuesToStringForExport()) + } + public static class SqlForm { + private Format format = Format.split; private Double apiVersion = null; private String schema; private String sql; private String sep = null; private String eol = null; - private boolean compact = false; private final Parameters parameters = new Parameters(); public Double getApiVersion() @@ -154,9 +173,19 @@ public Map getParameterMap() return parameters.map; } + public Format getFormat() + { + return format; + } + + public void setFormat(Format format) + { + this.format = format; + } + public String getSep() { - return null!=sep ? sep : compact ? "\u001f" : "\t"; + return null!=sep ? sep : format==Format.compact ? "\u001f" : "\t"; } public void setSep(String sep) @@ -167,7 +196,7 @@ public void setSep(String sep) public String getEol() { - return null!=eol ? eol : compact ? "\u001e" : "\t"; + return null!=eol ? eol : format==Format.compact ? "\u001e" : "\n"; } public void setEol(String eol) @@ -178,16 +207,22 @@ public void setEol(String eol) public boolean isCompact() { - return compact; + return Format.compact == format; } public void setCompact(boolean compact) { - this.compact = compact; + if (compact) + this.format = Format.compact; } } - + /// Execute a LabKey SQL query and return results as plain text. Designed for lightweight programmatic access without the overhead of QueryView/JSON API responses. + /// + /// Note this is still experimental as this API does not work well with some features. + /// In particular, some columns rely on custom DisplayColumn implmentations to return meaningful data, + /// and this code path does not use DisplayColumn. In particular group_concat result (e.g. multi-value foreign keys) + /// may not render correctly, as well as lineage columns like MaterialInputs/*. @RequiresPermission(ReadPermission.class) @Marshal(Marshaller.Jackson) public class ExecuteAction extends ReadOnlyApiAction @@ -239,10 +274,18 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio try (Results rs = QueryService.get().selectResults(schema, form.getSql(), null, form.getParameterMap(), true, false)) { getViewContext().getResponse().setContentType("text/plain"); - if (form.compact) - writeResults_compact(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); - else - writeResults_text(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); + switch (form.getFormat()) + { + case tsv: + writeResults_tsv(getViewContext().getResponse().getWriter(), rs); + break; + case split: + writeResults_text(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); + break; + case compact: + writeResults_compact(getViewContext().getResponse().getWriter(), rs, form.getSep(),form.getEol()); + break; + } } catch (QueryParseException x) { @@ -257,102 +300,135 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio } + JdbcType[] types; + Unit[] units; + + void initWriter(Results rs) throws SQLException + { + final int count = rs.getMetaData().getColumnCount(); + types = new JdbcType[count]; + units = new Unit[count]; + + for (int column = 1; column <= count; column++) + { + int index = column-1; + types[index] = JdbcType.valueOf(rs.getMetaData().getColumnType(column)); + ColumnInfo ci = rs.getColumn(column); + if (null != ci) + { + units[index] = ci.getDisplayUnit(); + } + } + } + + + void getStringData(Results rs, ArrayList out) throws SQLException + { + out.clear(); + for (int column = 1; column <= types.length; column++) + { + int index = column - 1; + String value = null; + + if (null != units[index]) + { + Number storageValue = types[index] == JdbcType.DECIMAL ? rs.getBigDecimal(column) : rs.getDouble(column); + if (!rs.wasNull()) + value = String.valueOf(units[index].fromStorageUnitValue(storageValue)); + } + else + { + switch (types[index]) + { + case TINYINT: + case SMALLINT: + case INTEGER: + { + int i = rs.getInt(column); + value = rs.wasNull() ? null : String.valueOf(i); + break; + } + case CHAR: + case VARCHAR: + case LONGVARCHAR: + { + value = rs.getString(column); + break; + } + case DOUBLE: + case REAL: + { + double d = rs.getDouble(column); + value = rs.wasNull() ? null : String.valueOf(d); + break; + } + case BOOLEAN: + { + boolean b = rs.getBoolean(column); + value = rs.wasNull() ? null : b ? "1" : "0"; + break; + } + case DECIMAL: + { + BigDecimal dec = rs.getBigDecimal(column); + value = null == dec ? null : dec.toPlainString(); + break; + } + case ARRAY: + { + Array array = rs.getArray(column); + if (null != array) + { + String[] strs = ConvertHelper.convert(array.getArray(), String[].class); + if (null != strs) + value = PageFlowUtil.joinValuesToStringForExport(List.of(strs)); + } + break; + } + default: + { + value = rs.getString(column); + break; + } + } + } + out.add(value); + } + } + + void writeResults_text(PrintWriter out, Results rs, String sep, String eol) throws SQLException { + initWriter(rs); final int count = rs.getMetaData().getColumnCount(); - final boolean serializeDateAsNumber=false; // meta-meta-data out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); - for (int i = 1; i <= count; i++) + for (int column = 1; column <= count; column++) { - out.write(rs.getColumn(i).getName()); - out.write(i == count ? eol : sep); + out.write(rs.getColumn(column).getName()); + out.write(column == count ? eol : sep); } - // pull types from ResultSetMetaData, not ColumnInfo - JdbcType[] types = new JdbcType[count + 1]; - for (int i = 1; i <= count; i++) + for (int column = 1; column <= count; column++) { - JdbcType jdbc = JdbcType.valueOf(rs.getMetaData().getColumnType(i)); - types[i] = jdbc; - out.write(jdbc.name()); - out.write(i == count ? eol : sep); + int index = column-1; + out.write(types[index].name()); + out.write(column == count ? eol : sep); } + ArrayList values = new ArrayList<>(count); + while (rs.next()) { - for (int column = 1; column <= count; column++) + getStringData(rs, values); + for (int index = 0; index < count; index++) { - // let's try to avoid tons of inspection if possible, and allocating tons of objects - // handle the most common types - printValue: - { - switch (types[column]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - { - int i = rs.getInt(column); - if (!rs.wasNull()) - out.print(i); - break printValue; - } - case CHAR: - case VARCHAR: - { - String s = rs.getString(column); - if (null != s) - out.write(s); - break printValue; - } - case DOUBLE: - case REAL: - { - double d = rs.getDouble(column); - if (!rs.wasNull()) - out.print(d); - break printValue; - } - case TIMESTAMP: - { - Date date = rs.getTimestamp(column); - if (null != date) - { - if (serializeDateAsNumber) - out.print(date.getTime()); - else - //out.write(DateUtil.formatJsonDateTime(date)); - out.write(DateUtil.formatIsoDateShortTime(date)); - } - break printValue; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - if (!rs.wasNull()) - out.write(b ? '1' : '0'); - break printValue; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - if (null != dec) - out.write(dec.toPlainString()); - break printValue; - } - default: - { - String obj = rs.getString(column); - if (null != obj) - out.write(obj); - break printValue; - } - } - } - out.write(column == count ? eol : sep); + String s = values.get(index); + if (null != s) + out.write(s); + out.write(index == count - 1 ? eol : sep); } } out.flush(); @@ -375,93 +451,45 @@ void writeResults_text(PrintWriter out, Results rs, String sep, String eol) thro */ void writeResults_compact(PrintWriter out, Results rs, String sep, String eol) throws SQLException { + initWriter(rs); final int count = rs.getMetaData().getColumnCount(); - final boolean serializeDateAsNumber=false; // meta-meta-data out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); - for (int i = 1; i <= count; i++) + for (int column = 1; column <= count; column++) { - out.write(rs.getColumn(i).getName()); - out.write(i == count ? eol : sep); + out.write(rs.getColumn(column).getName()); + out.write(column == count ? eol : sep); } - // pull types from ResultSetMetaData, not ColumnInfo - JdbcType[] types = new JdbcType[count + 1]; - for (int i = 1; i <= count; i++) + for (int index = 0; index < count; index++) { - JdbcType jdbc = JdbcType.valueOf(rs.getMetaData().getColumnType(i)); - types[i] = jdbc; - out.write(jdbc.name()); - out.write(i == count ? eol : sep); + out.write(types[index].name()); + out.write(index == count-1 ? eol : sep); } String DITTO = "\u0008"; - String[] prev = new String[count+1]; - String[] row = new String[count+1]; + ArrayList prev = new ArrayList<>(count); + ArrayList row = new ArrayList<>(count); while (rs.next()) { - for (int column = 1; column <= count; column++) - { - switch (types[column]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - case CHAR: - case VARCHAR: - case DOUBLE: - case REAL: - default: - { - row[column] = rs.getString(column); - break; - } - case TIMESTAMP: - { - Date date = rs.getTimestamp(column); - if (null == date) - row[column] = null; - else if (serializeDateAsNumber) - row[column] = Long.toString(date.getTime()); - else - { - String d = DateUtil.formatIsoDateShortTime(date); - if (d.endsWith(" 00:00")) - d = d.substring(0,d.length()-6); - row[column] = d; - } - break; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - row[column] = rs.wasNull() ? null : b ? "1": "0"; - break; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - row[column] = null==dec ? null : dec.toPlainString(); - break; - } - } - } - for (int column = 1; column <= count; column++) + getStringData(rs, row); + + for (int index = 0; index < count; index++) { - String s = row[column]; + String s = row.get(index); if (null != s && !s.isEmpty()) { - if (s.equals(prev[column])) + if (index < prev.size() && s.equals(prev.get(index))) out.write(DITTO); else out.write(s); } - out.write(column == count ? eol : sep); + out.write(index == count - 1 ? eol : sep); } - String[] t = prev; + ArrayList t = prev; prev = row; row = t; } @@ -469,6 +497,31 @@ else if (serializeDateAsNumber) } + /// export a Result set using RFC4180 formatting + /// use PageFlowUtil.joinValuesWithTabs4180 + void writeResults_tsv(PrintWriter out, Results rs) throws SQLException + { + initWriter(rs); + final int count = rs.getMetaData().getColumnCount(); + + List names = new ArrayList<>(count); + for (int column = 1; column <= count; column++) + names.add(rs.getColumn(column).getName()); + out.write(PageFlowUtil.joinValuesWithTabs4180(names)); + out.write('\n'); + + ArrayList values = new ArrayList<>(count); + + while (rs.next()) + { + getStringData(rs, values); + out.write(PageFlowUtil.joinValuesWithTabs4180(values)); + out.write('\n'); + } + out.flush(); + } + + public static class TestCase extends Assert { private static final String FOLDER_NAME = "sqlControllerTest"; @@ -525,15 +578,15 @@ public void tearDown() _folder = null; } - private MockHttpServletResponse executeSql(String schemaName, String sql, boolean compact) throws Exception + private MockHttpServletResponse executeSql(String schemaName, String sql, Format format) throws Exception { ActionURL url = new ActionURL("sql", "execute", _folder); if (schemaName != null) url.addParameter("schemaName", schemaName); if (sql != null) url.addParameter("sql", sql); - if (compact) - url.addParameter("compact", true); + if (null != format) + url.addParameter("format", format.name()); return ViewServlet.GET(url, TestContext.get().getUser(), null); } @@ -541,11 +594,11 @@ private MockHttpServletResponse executeSql(String schemaName, String sql, boolea public void testExecute_mssql() throws Exception { MockHttpServletResponse response = executeSql("lists", - "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", false); + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.split); assertEquals(HttpServletResponse.SC_OK, response.getStatus()); String content = response.getContentAsString(); - String[] tokens = content.split("\t"); + String[] tokens = content.split("[\t\n]"); // Header: meta-meta-data (3) + column names (3) + types (3) = 9 // Data: 3 rows * 3 columns = 9 @@ -574,6 +627,7 @@ public void testExecute_mssql() throws Exception assertEquals("Carol", tokens[15]); assertEquals("35", tokens[16]); } + @Test public void testExecute() throws Exception { @@ -584,11 +638,11 @@ public void testExecute() throws Exception } MockHttpServletResponse response = executeSql("lists", - "SELECT Name, Age, Score, Tags FROM " + LIST_NAME + " ORDER BY Name", false); + "SELECT Name, Age, Score, Tags FROM " + LIST_NAME + " ORDER BY Name", Format.split); assertEquals(HttpServletResponse.SC_OK, response.getStatus()); String content = response.getContentAsString(); - String[] tokens = content.split("\t"); + String[] tokens = content.split("[\t\n]"); // Header: meta-meta-data (3) + column names (4) + types (4) = 11 // Data: 3 rows * 4 columns = 12 @@ -627,7 +681,7 @@ public void testExecute() throws Exception public void testExecuteCompact() throws Exception { MockHttpServletResponse response = executeSql("lists", - "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", true); + "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", Format.compact); assertEquals(HttpServletResponse.SC_OK, response.getStatus()); String content = response.getContentAsString(); @@ -660,10 +714,44 @@ public void testExecuteCompact() throws Exception assertEquals("35", row3[1]); } + @Test + public void testExecuteTsv() throws Exception + { + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.tsv); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] lines = content.split("\n"); + assertTrue("Expected at least 4 lines (header + 3 data rows), got " + lines.length, lines.length >= 4); + + // Header row: column names + String[] headers = lines[0].split("\t"); + assertEquals("Name", headers[0]); + assertEquals("Age", headers[1]); + assertEquals("Score", headers[2]); + + // Data rows ordered by Name + String[] row1 = lines[1].split("\t"); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + assertEquals("95.5", row1[2]); + + String[] row2 = lines[2].split("\t"); + assertEquals("Bob", row2[0]); + assertEquals("30", row2[1]); + assertEquals("87.3", row2[2]); + + String[] row3 = lines[3].split("\t"); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + assertEquals("91.0", row3[2]); + } + @Test public void testNoSql() throws Exception { - MockHttpServletResponse response = executeSql("lists", null, false); + MockHttpServletResponse response = executeSql("lists", null, Format.split); assertTrue("Expected error about missing SQL", response.getContentAsString().contains("no sql provided")); } @@ -672,7 +760,7 @@ public void testNoSql() throws Exception public void testSchemaNotFound() throws Exception { MockHttpServletResponse response = executeSql("nonexistent", - "SELECT 1", false); + "SELECT 1", Format.tsv); assertTrue("Expected schema not found error", response.getContentAsString().contains("schema not found")); } From fccbd9cc73a65be5a07645457adc372c7d2fb06c Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 23 Apr 2026 13:24:26 -0700 Subject: [PATCH 04/10] review --- .../org/labkey/query/controllers/SqlController.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index 82d9c13ae20..b38e9dcde3e 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -99,7 +99,7 @@ public void set(String name, Object value) public enum Format { - split, // respose that can be parsed using String.split(), configure with 'sep' and 'eol' + split, // response that can be parsed using String.split(), configure with 'sep' and 'eol' compact, // same as split, but with ditto markers (cheap way to compress and save space on client) @@ -220,7 +220,7 @@ public void setCompact(boolean compact) /// Execute a LabKey SQL query and return results as plain text. Designed for lightweight programmatic access without the overhead of QueryView/JSON API responses. /// /// Note this is still experimental as this API does not work well with some features. - /// In particular, some columns rely on custom DisplayColumn implmentations to return meaningful data, + /// In particular, some columns rely on custom DisplayColumn implementations to return meaningful data, /// and this code path does not use DisplayColumn. In particular group_concat result (e.g. multi-value foreign keys) /// may not render correctly, as well as lineage columns like MaterialInputs/*. @RequiresPermission(ReadPermission.class) @@ -622,10 +622,13 @@ public void testExecute_mssql() throws Exception // Data rows ordered by Name assertEquals("Alice", tokens[9]); assertEquals("30", tokens[10]); + assertEquals("95.5", tokens[11]); assertEquals("Bob", tokens[12]); assertEquals("30", tokens[13]); + assertEquals("87.3", tokens[14]); assertEquals("Carol", tokens[15]); assertEquals("35", tokens[16]); + assertEquals("91.0", tokens[17]); } @Test @@ -668,12 +671,15 @@ public void testExecute() throws Exception // Data rows ordered by Name (4 columns per row) assertEquals("Alice", tokens[11]); assertEquals("30", tokens[12]); + assertEquals("95.5", tokens[13]); assertTrue("Alice Tags", tokens[14].contains("Red") && tokens[14].contains("Green")); assertEquals("Bob", tokens[15]); assertEquals("30", tokens[16]); + assertEquals("87.3", tokens[17]); assertTrue("Bob Tags", tokens[18].contains("Blue")); assertEquals("Carol", tokens[19]); assertEquals("35", tokens[20]); + assertEquals("91.0", tokens[21]); assertTrue("Carol Tags", tokens[22].contains("Red") && tokens[22].contains("Blue") && tokens[22].contains("Green")); } @@ -765,4 +771,4 @@ public void testSchemaNotFound() throws Exception response.getContentAsString().contains("schema not found")); } } -} \ No newline at end of file +} From 6fd5af022eee754f05114c3794406b87efcd200a Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 14:09:19 -0700 Subject: [PATCH 05/10] Execute2Action checkpoint version that uses DisplayColumn to handle "interesting" columns. --- api/src/org/labkey/api/data/DataColumn.java | 6 + .../org/labkey/api/data/DisplayColumn.java | 5 + .../query/controllers/SqlController.java | 592 +++++++++++------- 3 files changed, 388 insertions(+), 215 deletions(-) diff --git a/api/src/org/labkey/api/data/DataColumn.java b/api/src/org/labkey/api/data/DataColumn.java index 2bdbffe6c73..17810bec5c7 100644 --- a/api/src/org/labkey/api/data/DataColumn.java +++ b/api/src/org/labkey/api/data/DataColumn.java @@ -191,6 +191,12 @@ protected ColumnInfo getDisplayField(@NotNull ColumnInfo col, boolean withLookup return null==display ? col : display; } + @Override + public void setWithLookup(boolean withLookup) + { + _displayColumn = withLookup ? getDisplayField(_boundColumn, true) : _boundColumn; + } + @Override public String toString() { diff --git a/api/src/org/labkey/api/data/DisplayColumn.java b/api/src/org/labkey/api/data/DisplayColumn.java index 83c18d15813..0b9cebd9a21 100644 --- a/api/src/org/labkey/api/data/DisplayColumn.java +++ b/api/src/org/labkey/api/data/DisplayColumn.java @@ -1232,6 +1232,11 @@ public void setRequiresHtmlFiltering(boolean requiresHtmlFiltering) _requiresHtmlFiltering = requiresHtmlFiltering; } + public void setWithLookup(boolean withLookup) + { + // subclasses override as needed + } + public void setLinkTarget(String linkTarget) { _linkTarget = linkTarget; diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index b38e9dcde3e..8969a04ad33 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -29,13 +29,16 @@ import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.ResultSetRowMapFactory; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DisplayColumn; import org.labkey.api.data.JdbcType; import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RenderContext; import org.labkey.api.data.Results; import org.labkey.api.data.TableInfo; import org.labkey.api.exp.PropertyType; @@ -185,7 +188,7 @@ public void setFormat(Format format) public String getSep() { - return null!=sep ? sep : format==Format.compact ? "\u001f" : "\t"; + return null!=sep ? sep : format==Format.compact ? "" : "\t"; } public void setSep(String sep) @@ -196,7 +199,7 @@ public void setSep(String sep) public String getEol() { - return null!=eol ? eol : format==Format.compact ? "\u001e" : "\n"; + return null!=eol ? eol : format==Format.compact ? "" : "\n"; } public void setEol(String eol) @@ -227,6 +230,224 @@ public void setCompact(boolean compact) @Marshal(Marshaller.Jackson) public class ExecuteAction extends ReadOnlyApiAction { + JdbcType[] types; + Unit[] units; + + void initWriter(Results rs) throws SQLException + { + final int count = rs.getMetaData().getColumnCount(); + types = new JdbcType[count]; + units = new Unit[count]; + + for (int column = 1; column <= count; column++) + { + int index = column-1; + types[index] = JdbcType.valueOf(rs.getMetaData().getColumnType(column)); + ColumnInfo ci = rs.getColumn(column); + if (null != ci) + { + units[index] = ci.getDisplayUnit(); + } + } + } + + void getStringData(Results rs, ArrayList out) throws SQLException + { + out.clear(); + for (int column = 1; column <= types.length; column++) + { + int index = column - 1; + String value = null; + + if (null != units[index]) + { + Number storageValue = types[index] == JdbcType.DECIMAL ? rs.getBigDecimal(column) : rs.getDouble(column); + if (!rs.wasNull()) + value = String.valueOf(units[index].fromStorageUnitValue(storageValue)); + } + else + { + switch (types[index]) + { + case TINYINT: + case SMALLINT: + case INTEGER: + { + int i = rs.getInt(column); + value = rs.wasNull() ? null : String.valueOf(i); + break; + } + case CHAR: + case VARCHAR: + case LONGVARCHAR: + { + value = rs.getString(column); + break; + } + case DOUBLE: + case REAL: + { + double d = rs.getDouble(column); + value = rs.wasNull() ? null : String.valueOf(d); + break; + } + case BOOLEAN: + { + boolean b = rs.getBoolean(column); + value = rs.wasNull() ? null : b ? "1" : "0"; + break; + } + case DECIMAL: + { + BigDecimal dec = rs.getBigDecimal(column); + value = null == dec ? null : dec.toPlainString(); + break; + } + case ARRAY: + { + Array array = rs.getArray(column); + if (null != array) + { + String[] strs = ConvertHelper.convert(array.getArray(), String[].class); + if (null != strs) + value = PageFlowUtil.joinValuesToStringForExport(List.of(strs)); + } + break; + } + default: + { + value = rs.getString(column); + break; + } + } + } + out.add(value); + } + } + + void writeResults_text(PrintWriter out, Results rs, String sep, String eol) throws SQLException + { + initWriter(rs); + final int count = rs.getMetaData().getColumnCount(); + + // meta-meta-data + out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + + for (int column = 1; column <= count; column++) + { + out.write(rs.getColumn(column).getName()); + out.write(column == count ? eol : sep); + } + + for (int column = 1; column <= count; column++) + { + int index = column-1; + out.write(types[index].name()); + out.write(column == count ? eol : sep); + } + + ArrayList values = new ArrayList<>(count); + + while (rs.next()) + { + getStringData(rs, values); + for (int index = 0; index < count; index++) + { + String s = values.get(index); + if (null != s) + out.write(s); + out.write(index == count - 1 ? eol : sep); + } + } + out.flush(); + } + + /** + * try to generate a more compact representation + * + * the value 0x1A SUB means same value as previous row + * truncate trailing time from date-only values + * + * Consider disambiguating sql NULL and empty string. NULL is more common so using a shorter + * encoding for NULL and longer for empty string makes sense (e.g. "\u0000") + * + * TODO: binary/blob length prefixed encoding + * + * Note that while writeResults_text tries to not generate extra Strings to GC, I think the actual PrintWriter + * implementation is generating strings inside out.write(). So this is probably not much different from a GC + * perspective. + */ + void writeResults_compact(PrintWriter out, Results rs, String sep, String eol) throws SQLException + { + initWriter(rs); + final int count = rs.getMetaData().getColumnCount(); + + // meta-meta-data + out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); + + for (int column = 1; column <= count; column++) + { + out.write(rs.getColumn(column).getName()); + out.write(column == count ? eol : sep); + } + + for (int index = 0; index < count; index++) + { + out.write(types[index].name()); + out.write(index == count-1 ? eol : sep); + } + + String DITTO = ""; + ArrayList prev = new ArrayList<>(count); + ArrayList row = new ArrayList<>(count); + + while (rs.next()) + { + getStringData(rs, row); + + for (int index = 0; index < count; index++) + { + String s = row.get(index); + if (null != s && !s.isEmpty()) + { + if (index < prev.size() && s.equals(prev.get(index))) + out.write(DITTO); + else + out.write(s); + } + out.write(index == count - 1 ? eol : sep); + } + ArrayList t = prev; + prev = row; + row = t; + } + out.flush(); + } + + /// export a Result set using RFC4180 formatting + /// use PageFlowUtil.joinValuesWithTabs4180 + void writeResults_tsv(PrintWriter out, Results rs) throws SQLException + { + initWriter(rs); + final int count = rs.getMetaData().getColumnCount(); + + List names = new ArrayList<>(count); + for (int column = 1; column <= count; column++) + names.add(rs.getColumn(column).getName()); + out.write(PageFlowUtil.joinValuesWithTabs4180(names)); + out.write('\n'); + + ArrayList values = new ArrayList<>(count); + + while (rs.next()) + { + getStringData(rs, values); + out.write(PageFlowUtil.joinValuesWithTabs4180(values)); + out.write('\n'); + } + out.flush(); + } + @Override public Object execute(SqlForm form, BindException errors) throws ServletException { @@ -300,225 +521,45 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio } - JdbcType[] types; - Unit[] units; - - void initWriter(Results rs) throws SQLException - { - final int count = rs.getMetaData().getColumnCount(); - types = new JdbcType[count]; - units = new Unit[count]; - - for (int column = 1; column <= count; column++) - { - int index = column-1; - types[index] = JdbcType.valueOf(rs.getMetaData().getColumnType(column)); - ColumnInfo ci = rs.getColumn(column); - if (null != ci) - { - units[index] = ci.getDisplayUnit(); - } - } - } - - - void getStringData(Results rs, ArrayList out) throws SQLException - { - out.clear(); - for (int column = 1; column <= types.length; column++) - { - int index = column - 1; - String value = null; - - if (null != units[index]) - { - Number storageValue = types[index] == JdbcType.DECIMAL ? rs.getBigDecimal(column) : rs.getDouble(column); - if (!rs.wasNull()) - value = String.valueOf(units[index].fromStorageUnitValue(storageValue)); - } - else - { - switch (types[index]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - { - int i = rs.getInt(column); - value = rs.wasNull() ? null : String.valueOf(i); - break; - } - case CHAR: - case VARCHAR: - case LONGVARCHAR: - { - value = rs.getString(column); - break; - } - case DOUBLE: - case REAL: - { - double d = rs.getDouble(column); - value = rs.wasNull() ? null : String.valueOf(d); - break; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - value = rs.wasNull() ? null : b ? "1" : "0"; - break; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - value = null == dec ? null : dec.toPlainString(); - break; - } - case ARRAY: - { - Array array = rs.getArray(column); - if (null != array) - { - String[] strs = ConvertHelper.convert(array.getArray(), String[].class); - if (null != strs) - value = PageFlowUtil.joinValuesToStringForExport(List.of(strs)); - } - break; - } - default: - { - value = rs.getString(column); - break; - } - } - } - out.add(value); - } - } - - - void writeResults_text(PrintWriter out, Results rs, String sep, String eol) throws SQLException - { - initWriter(rs); - final int count = rs.getMetaData().getColumnCount(); - - // meta-meta-data - out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); - - for (int column = 1; column <= count; column++) - { - out.write(rs.getColumn(column).getName()); - out.write(column == count ? eol : sep); - } - - for (int column = 1; column <= count; column++) - { - int index = column-1; - out.write(types[index].name()); - out.write(column == count ? eol : sep); - } - - ArrayList values = new ArrayList<>(count); - - while (rs.next()) - { - getStringData(rs, values); - for (int index = 0; index < count; index++) - { - String s = values.get(index); - if (null != s) - out.write(s); - out.write(index == count - 1 ? eol : sep); - } - } - out.flush(); - } - - /** - * try to generate a more compact representation - * - * the value 0x1A SUB means same value as previous row - * truncate trailing time from date-only values - * - * Consider disambiguating sql NULL and empty string. NULL is more common so using a shorter - * encoding for NULL and longer for empty string makes sense (e.g. "\u0000") - * - * TODO: binary/blob length prefixed encoding - * - * Note that while writeResults_text tries to not generate extra Strings to GC, I think the actual PrintWriter - * implementation is generating strings inside out.write(). So this is probably not much different from a GC - * perspective. - */ - void writeResults_compact(PrintWriter out, Results rs, String sep, String eol) throws SQLException + /// Like ExecuteAction but routes value rendering through DisplayColumn.getTsvFormattedValue() for correct + /// type dispatch (e.g. dates as ISO-8601, multi-value columns via their DisplayColumn subclass). + @RequiresPermission(ReadPermission.class) + @Marshal(Marshaller.Jackson) + public class Execute2Action extends ExecuteAction { - initWriter(rs); - final int count = rs.getMetaData().getColumnCount(); - - // meta-meta-data - out.write("18.2"+sep+"name"+sep+"jdbcType"+eol); - - for (int column = 1; column <= count; column++) - { - out.write(rs.getColumn(column).getName()); - out.write(column == count ? eol : sep); - } + DisplayColumn[] dcs; + RenderContext renderCtx; + ResultSetRowMapFactory rowMapFactory; - for (int index = 0; index < count; index++) - { - out.write(types[index].name()); - out.write(index == count-1 ? eol : sep); - } - - String DITTO = "\u0008"; - ArrayList prev = new ArrayList<>(count); - ArrayList row = new ArrayList<>(count); - - while (rs.next()) + @Override + void initWriter(Results rs) throws SQLException { - getStringData(rs, row); - - for (int index = 0; index < count; index++) + super.initWriter(rs); + final int count = rs.getMetaData().getColumnCount(); + dcs = new DisplayColumn[count]; + for (int column = 1; column <= count; column++) { - String s = row.get(index); - if (null != s && !s.isEmpty()) - { - if (index < prev.size() && s.equals(prev.get(index))) - out.write(DITTO); - else - out.write(s); - } - out.write(index == count - 1 ? eol : sep); + ColumnInfo col = rs.getColumn(column); + DisplayColumn dc = col.getDisplayColumnFactory().createRenderer(col); + dc.setWithLookup(false); + dc.setFormatString(null); + dc.setTsvFormatString(null); + dc.setRequiresHtmlFiltering(false); + dcs[column - 1] = dc; } - ArrayList t = prev; - prev = row; - row = t; + renderCtx = new RenderContext(getViewContext()); + renderCtx.setResults(rs); + rowMapFactory = ResultSetRowMapFactory.create(rs); } - out.flush(); - } - - - /// export a Result set using RFC4180 formatting - /// use PageFlowUtil.joinValuesWithTabs4180 - void writeResults_tsv(PrintWriter out, Results rs) throws SQLException - { - initWriter(rs); - final int count = rs.getMetaData().getColumnCount(); - List names = new ArrayList<>(count); - for (int column = 1; column <= count; column++) - names.add(rs.getColumn(column).getName()); - out.write(PageFlowUtil.joinValuesWithTabs4180(names)); - out.write('\n'); - - ArrayList values = new ArrayList<>(count); - - while (rs.next()) + @Override + void getStringData(Results rs, ArrayList out) throws SQLException { - getStringData(rs, values); - out.write(PageFlowUtil.joinValuesWithTabs4180(values)); - out.write('\n'); + out.clear(); + renderCtx.setRow(rowMapFactory.getRowMap(rs)); + for (DisplayColumn dc : dcs) + out.add(dc.getTsvFormattedValue(renderCtx)); } - out.flush(); } @@ -691,9 +732,9 @@ public void testExecuteCompact() throws Exception assertEquals(HttpServletResponse.SC_OK, response.getStatus()); String content = response.getContentAsString(); - String sep = "\u001f"; - String eol = "\u001e"; - String ditto = "\u0008"; + String sep = ""; + String eol = ""; + String ditto = ""; String[] records = content.split(eol); // records: [0]=meta, [1]=column names, [2]=types, [3..5]=data rows @@ -770,5 +811,126 @@ public void testSchemaNotFound() throws Exception assertTrue("Expected schema not found error", response.getContentAsString().contains("schema not found")); } + + private MockHttpServletResponse executeSql2(String schemaName, String sql, Format format) throws Exception + { + ActionURL url = new ActionURL("sql", "execute2", _folder); + if (schemaName != null) + url.addParameter("schemaName", schemaName); + if (sql != null) + url.addParameter("sql", sql); + if (null != format) + url.addParameter("format", format.name()); + return ViewServlet.GET(url, TestContext.get().getUser(), null); + } + + @Test + public void testExecute2() throws Exception + { + // Basic split format: same data shape as testExecute_mssql + MockHttpServletResponse response = executeSql2("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] tokens = content.split("[\t\n]"); + + assertTrue("Expected at least 18 tokens, got " + tokens.length, tokens.length >= 18); + assertEquals("18.2", tokens[0]); + assertEquals("name", tokens[1]); + assertEquals("jdbcType", tokens[2]); + assertEquals("Name", tokens[3]); + assertEquals("Age", tokens[4]); + assertEquals("Score", tokens[5]); + assertEquals("VARCHAR", tokens[6]); + assertEquals("INTEGER", tokens[7]); + assertEquals("DOUBLE", tokens[8]); + assertEquals("Alice", tokens[9]); + assertEquals("30", tokens[10]); + assertEquals("95.5", tokens[11]); + assertEquals("Bob", tokens[12]); + assertEquals("30", tokens[13]); + assertEquals("87.3", tokens[14]); + assertEquals("Carol", tokens[15]); + assertEquals("35", tokens[16]); + assertEquals("91.0", tokens[17]); + + // Date column should render as ISO-8601, not a raw JDBC string + MockHttpServletResponse dateResponse = executeSql2("lists", + "SELECT Name, Created FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, dateResponse.getStatus()); + String dateContent = dateResponse.getContentAsString(); + String[] dateTokens = dateContent.split("[\t\n]"); + // tokens: meta(3) + colNames(2) + jdbcTypes(2) + data(3 rows * 2 cols = 6) = 13 minimum + assertTrue("Expected at least 13 date tokens, got " + dateTokens.length, dateTokens.length >= 13); + assertEquals("Alice", dateTokens[7]); + // we use space instead of 'T' + assertTrue("Created date should be ISO-8601: " + dateTokens[8], + dateTokens[8].matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*")); + } + + @Test + public void testExecuteCompact2() throws Exception + { + MockHttpServletResponse response = executeSql2("lists", + "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", Format.compact); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String sep = ""; + String eol = ""; + String ditto = ""; + + String[] records = content.split(eol); + assertTrue("Expected at least 6 records", records.length >= 6); + + String[] colNames = records[1].split(sep); + assertEquals("Name", colNames[0]); + assertEquals("Age", colNames[1]); + + String[] row1 = records[3].split(sep, -1); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + + String[] row2 = records[4].split(sep, -1); + assertEquals("Bob", row2[0]); + assertEquals(ditto, row2[1]); + + String[] row3 = records[5].split(sep, -1); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + } + + @Test + public void testExecuteTsv2() throws Exception + { + MockHttpServletResponse response = executeSql2("lists", + "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.tsv); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + + String content = response.getContentAsString(); + String[] lines = content.split("\n"); + assertTrue("Expected at least 4 lines (header + 3 data rows), got " + lines.length, lines.length >= 4); + + String[] headers = lines[0].split("\t"); + assertEquals("Name", headers[0]); + assertEquals("Age", headers[1]); + assertEquals("Score", headers[2]); + + String[] row1 = lines[1].split("\t"); + assertEquals("Alice", row1[0]); + assertEquals("30", row1[1]); + assertEquals("95.5", row1[2]); + + String[] row2 = lines[2].split("\t"); + assertEquals("Bob", row2[0]); + assertEquals("30", row2[1]); + assertEquals("87.3", row2[2]); + + String[] row3 = lines[3].split("\t"); + assertEquals("Carol", row3[0]); + assertEquals("35", row3[1]); + assertEquals("91.0", row3[2]); + } } } From ae5c43595e561904cf5ca38916333f97cef2dfe0 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 15:30:05 -0700 Subject: [PATCH 06/10] Execute2Action --- .../src/org/labkey/query/controllers/LabKeySql.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/query/src/org/labkey/query/controllers/LabKeySql.md b/query/src/org/labkey/query/controllers/LabKeySql.md index b02b11d1261..08b77ffc125 100644 --- a/query/src/org/labkey/query/controllers/LabKeySql.md +++ b/query/src/org/labkey/query/controllers/LabKeySql.md @@ -336,4 +336,17 @@ When writing LabKey SQL queries that work with JSON columns: 3. **Use `jsonb_extract_path_text()` for nested field access** — this is often the clearest way to extract a deeply nested text value: `jsonb_extract_path_text(col, 'level1', 'level2', 'field')`. 4. **Use `jsonb_build_object()` to construct JSON** — for building JSON from column values: `jsonb_build_object('id', rowid, 'name', label)`. 5. **Check database type first** — these functions only work on PostgreSQL. If the target server may use MS SQL Server, do not use them. -6. **The `validateSQL` MCP tool can verify syntax** — use it to check JSON function calls before the user saves a query. \ No newline at end of file +6. **The `validateSQL` MCP tool can verify syntax** — use it to check JSON function calls before the user saves a query. + +----- + +### **Fetching Live Data for LLM Inspection** + +The `sql-execute2.view` endpoint is for **the LLM to inspect live data while generating code** — do not emit it in generated Python, R, or other scripts (use the `labkey` client API there instead). Always include `LIMIT` to avoid fetching excess rows. + +```bash +curl -H "Authorization: Bearer " \ + "https:////sql-execute2.view?schemaName=&sql=SELECT+...+LIMIT+20&format=tsv" +``` + +The response is RFC 4180 TSV: a header row of column names followed by one data row per result. Values are rendered via `DisplayColumn.getTsvFormattedValue()` — dates as ISO-8601, multi-value columns correctly serialized. \ No newline at end of file From c04d95f31866a3ffc9b66f952a9476949f671b49 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 16:02:58 -0700 Subject: [PATCH 07/10] Execute2Action --- query/src/org/labkey/query/controllers/LabKeySql.md | 1 + 1 file changed, 1 insertion(+) diff --git a/query/src/org/labkey/query/controllers/LabKeySql.md b/query/src/org/labkey/query/controllers/LabKeySql.md index 08b77ffc125..c72e19ec8e8 100644 --- a/query/src/org/labkey/query/controllers/LabKeySql.md +++ b/query/src/org/labkey/query/controllers/LabKeySql.md @@ -343,6 +343,7 @@ When writing LabKey SQL queries that work with JSON columns: ### **Fetching Live Data for LLM Inspection** The `sql-execute2.view` endpoint is for **the LLM to inspect live data while generating code** — do not emit it in generated Python, R, or other scripts (use the `labkey` client API there instead). Always include `LIMIT` to avoid fetching excess rows. +Unlike the LabKey client APIs (`selectRows`, `executeSql`), this endpoint does not automatically resolve lookups to display values — it returns exactly what the SQL selects. Use LabKey SQL's dot-notation to traverse lookups explicitly when you need a human-readable value. For example, `SELECT CreatedBy FROM lists.MyList` returns a raw integer user ID; `SELECT CreatedBy.DisplayName FROM lists.MyList` returns the display name. ```bash curl -H "Authorization: Bearer " \ From 5ed32be11da404cb399e07a4745300c4f7a2e31d Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 16:33:27 -0700 Subject: [PATCH 08/10] replace origial action with DisplayColumn based action --- .../query/controllers/SqlController.java | 279 +++--------------- 1 file changed, 36 insertions(+), 243 deletions(-) diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index 8969a04ad33..5dcaba6cb3e 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -69,8 +69,6 @@ import java.io.IOException; import java.io.PrintWriter; -import java.math.BigDecimal; -import java.sql.Array; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -221,108 +219,49 @@ public void setCompact(boolean compact) } /// Execute a LabKey SQL query and return results as plain text. Designed for lightweight programmatic access without the overhead of QueryView/JSON API responses. - /// - /// Note this is still experimental as this API does not work well with some features. - /// In particular, some columns rely on custom DisplayColumn implementations to return meaningful data, - /// and this code path does not use DisplayColumn. In particular group_concat result (e.g. multi-value foreign keys) - /// may not render correctly, as well as lineage columns like MaterialInputs/*. + /// This action routes value rendering through DisplayColumn.getTsvFormattedValue() for correct + /// type dispatch (e.g. dates as ISO-8601, multi-value columns via their DisplayColumn subclass). @RequiresPermission(ReadPermission.class) @Marshal(Marshaller.Jackson) public class ExecuteAction extends ReadOnlyApiAction { JdbcType[] types; Unit[] units; + DisplayColumn[] dcs; + RenderContext renderCtx; + ResultSetRowMapFactory rowMapFactory; void initWriter(Results rs) throws SQLException { final int count = rs.getMetaData().getColumnCount(); types = new JdbcType[count]; units = new Unit[count]; + dcs = new DisplayColumn[count]; for (int column = 1; column <= count; column++) { - int index = column-1; + int index = column - 1; types[index] = JdbcType.valueOf(rs.getMetaData().getColumnType(column)); - ColumnInfo ci = rs.getColumn(column); - if (null != ci) - { - units[index] = ci.getDisplayUnit(); - } + ColumnInfo col = rs.getColumn(column); + units[index] = col.getDisplayUnit(); + DisplayColumn dc = col.getDisplayColumnFactory().createRenderer(col); + dc.setWithLookup(false); + dc.setFormatString(null); + dc.setTsvFormatString(null); + dc.setRequiresHtmlFiltering(false); + dcs[index] = dc; } + renderCtx = new RenderContext(getViewContext()); + renderCtx.setResults(rs); + rowMapFactory = ResultSetRowMapFactory.create(rs); } void getStringData(Results rs, ArrayList out) throws SQLException { out.clear(); - for (int column = 1; column <= types.length; column++) - { - int index = column - 1; - String value = null; - - if (null != units[index]) - { - Number storageValue = types[index] == JdbcType.DECIMAL ? rs.getBigDecimal(column) : rs.getDouble(column); - if (!rs.wasNull()) - value = String.valueOf(units[index].fromStorageUnitValue(storageValue)); - } - else - { - switch (types[index]) - { - case TINYINT: - case SMALLINT: - case INTEGER: - { - int i = rs.getInt(column); - value = rs.wasNull() ? null : String.valueOf(i); - break; - } - case CHAR: - case VARCHAR: - case LONGVARCHAR: - { - value = rs.getString(column); - break; - } - case DOUBLE: - case REAL: - { - double d = rs.getDouble(column); - value = rs.wasNull() ? null : String.valueOf(d); - break; - } - case BOOLEAN: - { - boolean b = rs.getBoolean(column); - value = rs.wasNull() ? null : b ? "1" : "0"; - break; - } - case DECIMAL: - { - BigDecimal dec = rs.getBigDecimal(column); - value = null == dec ? null : dec.toPlainString(); - break; - } - case ARRAY: - { - Array array = rs.getArray(column); - if (null != array) - { - String[] strs = ConvertHelper.convert(array.getArray(), String[].class); - if (null != strs) - value = PageFlowUtil.joinValuesToStringForExport(List.of(strs)); - } - break; - } - default: - { - value = rs.getString(column); - break; - } - } - } - out.add(value); - } + renderCtx.setRow(rowMapFactory.getRowMap(rs)); + for (DisplayColumn dc : dcs) + out.add(dc.getTsvFormattedValue(renderCtx)); } void writeResults_text(PrintWriter out, Results rs, String sep, String eol) throws SQLException @@ -521,47 +460,6 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio } - /// Like ExecuteAction but routes value rendering through DisplayColumn.getTsvFormattedValue() for correct - /// type dispatch (e.g. dates as ISO-8601, multi-value columns via their DisplayColumn subclass). - @RequiresPermission(ReadPermission.class) - @Marshal(Marshaller.Jackson) - public class Execute2Action extends ExecuteAction - { - DisplayColumn[] dcs; - RenderContext renderCtx; - ResultSetRowMapFactory rowMapFactory; - - @Override - void initWriter(Results rs) throws SQLException - { - super.initWriter(rs); - final int count = rs.getMetaData().getColumnCount(); - dcs = new DisplayColumn[count]; - for (int column = 1; column <= count; column++) - { - ColumnInfo col = rs.getColumn(column); - DisplayColumn dc = col.getDisplayColumnFactory().createRenderer(col); - dc.setWithLookup(false); - dc.setFormatString(null); - dc.setTsvFormatString(null); - dc.setRequiresHtmlFiltering(false); - dcs[column - 1] = dc; - } - renderCtx = new RenderContext(getViewContext()); - renderCtx.setResults(rs); - rowMapFactory = ResultSetRowMapFactory.create(rs); - } - - @Override - void getStringData(Results rs, ArrayList out) throws SQLException - { - out.clear(); - renderCtx.setRow(rowMapFactory.getRowMap(rs)); - for (DisplayColumn dc : dcs) - out.add(dc.getTsvFormattedValue(renderCtx)); - } - } - public static class TestCase extends Assert { @@ -795,6 +693,21 @@ public void testExecuteTsv() throws Exception assertEquals("91.0", row3[2]); } + @Test + public void testExecuteDate() throws Exception + { + // Date columns should render as ISO-8601 (space instead of 'T') via DisplayColumn.getTsvFormattedValue() + MockHttpServletResponse response = executeSql("lists", + "SELECT Name, Created FROM " + LIST_NAME + " ORDER BY Name", Format.split); + assertEquals(HttpServletResponse.SC_OK, response.getStatus()); + String[] tokens = response.getContentAsString().split("[\t\n]"); + // tokens: meta(3) + colNames(2) + jdbcTypes(2) + data(3 rows * 2 cols = 6) = 13 minimum + assertTrue("Expected at least 13 tokens, got " + tokens.length, tokens.length >= 13); + assertEquals("Alice", tokens[7]); + assertTrue("Created date should be ISO-8601: " + tokens[8], + tokens[8].matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*")); + } + @Test public void testNoSql() throws Exception { @@ -812,125 +725,5 @@ public void testSchemaNotFound() throws Exception response.getContentAsString().contains("schema not found")); } - private MockHttpServletResponse executeSql2(String schemaName, String sql, Format format) throws Exception - { - ActionURL url = new ActionURL("sql", "execute2", _folder); - if (schemaName != null) - url.addParameter("schemaName", schemaName); - if (sql != null) - url.addParameter("sql", sql); - if (null != format) - url.addParameter("format", format.name()); - return ViewServlet.GET(url, TestContext.get().getUser(), null); - } - - @Test - public void testExecute2() throws Exception - { - // Basic split format: same data shape as testExecute_mssql - MockHttpServletResponse response = executeSql2("lists", - "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.split); - assertEquals(HttpServletResponse.SC_OK, response.getStatus()); - - String content = response.getContentAsString(); - String[] tokens = content.split("[\t\n]"); - - assertTrue("Expected at least 18 tokens, got " + tokens.length, tokens.length >= 18); - assertEquals("18.2", tokens[0]); - assertEquals("name", tokens[1]); - assertEquals("jdbcType", tokens[2]); - assertEquals("Name", tokens[3]); - assertEquals("Age", tokens[4]); - assertEquals("Score", tokens[5]); - assertEquals("VARCHAR", tokens[6]); - assertEquals("INTEGER", tokens[7]); - assertEquals("DOUBLE", tokens[8]); - assertEquals("Alice", tokens[9]); - assertEquals("30", tokens[10]); - assertEquals("95.5", tokens[11]); - assertEquals("Bob", tokens[12]); - assertEquals("30", tokens[13]); - assertEquals("87.3", tokens[14]); - assertEquals("Carol", tokens[15]); - assertEquals("35", tokens[16]); - assertEquals("91.0", tokens[17]); - - // Date column should render as ISO-8601, not a raw JDBC string - MockHttpServletResponse dateResponse = executeSql2("lists", - "SELECT Name, Created FROM " + LIST_NAME + " ORDER BY Name", Format.split); - assertEquals(HttpServletResponse.SC_OK, dateResponse.getStatus()); - String dateContent = dateResponse.getContentAsString(); - String[] dateTokens = dateContent.split("[\t\n]"); - // tokens: meta(3) + colNames(2) + jdbcTypes(2) + data(3 rows * 2 cols = 6) = 13 minimum - assertTrue("Expected at least 13 date tokens, got " + dateTokens.length, dateTokens.length >= 13); - assertEquals("Alice", dateTokens[7]); - // we use space instead of 'T' - assertTrue("Created date should be ISO-8601: " + dateTokens[8], - dateTokens[8].matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.*")); - } - - @Test - public void testExecuteCompact2() throws Exception - { - MockHttpServletResponse response = executeSql2("lists", - "SELECT Name, Age FROM " + LIST_NAME + " ORDER BY Age, Name", Format.compact); - assertEquals(HttpServletResponse.SC_OK, response.getStatus()); - - String content = response.getContentAsString(); - String sep = ""; - String eol = ""; - String ditto = ""; - - String[] records = content.split(eol); - assertTrue("Expected at least 6 records", records.length >= 6); - - String[] colNames = records[1].split(sep); - assertEquals("Name", colNames[0]); - assertEquals("Age", colNames[1]); - - String[] row1 = records[3].split(sep, -1); - assertEquals("Alice", row1[0]); - assertEquals("30", row1[1]); - - String[] row2 = records[4].split(sep, -1); - assertEquals("Bob", row2[0]); - assertEquals(ditto, row2[1]); - - String[] row3 = records[5].split(sep, -1); - assertEquals("Carol", row3[0]); - assertEquals("35", row3[1]); - } - - @Test - public void testExecuteTsv2() throws Exception - { - MockHttpServletResponse response = executeSql2("lists", - "SELECT Name, Age, Score FROM " + LIST_NAME + " ORDER BY Name", Format.tsv); - assertEquals(HttpServletResponse.SC_OK, response.getStatus()); - - String content = response.getContentAsString(); - String[] lines = content.split("\n"); - assertTrue("Expected at least 4 lines (header + 3 data rows), got " + lines.length, lines.length >= 4); - - String[] headers = lines[0].split("\t"); - assertEquals("Name", headers[0]); - assertEquals("Age", headers[1]); - assertEquals("Score", headers[2]); - - String[] row1 = lines[1].split("\t"); - assertEquals("Alice", row1[0]); - assertEquals("30", row1[1]); - assertEquals("95.5", row1[2]); - - String[] row2 = lines[2].split("\t"); - assertEquals("Bob", row2[0]); - assertEquals("30", row2[1]); - assertEquals("87.3", row2[2]); - - String[] row3 = lines[3].split("\t"); - assertEquals("Carol", row3[0]); - assertEquals("35", row3[1]); - assertEquals("91.0", row3[2]); - } } } From bf2e8f7cb69d60a942517d005254b2eb11321415 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 16:35:49 -0700 Subject: [PATCH 09/10] lint --- query/src/org/labkey/query/controllers/SqlController.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/query/src/org/labkey/query/controllers/SqlController.java b/query/src/org/labkey/query/controllers/SqlController.java index 5dcaba6cb3e..5c4a2267991 100644 --- a/query/src/org/labkey/query/controllers/SqlController.java +++ b/query/src/org/labkey/query/controllers/SqlController.java @@ -19,6 +19,7 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.junit.After; import org.junit.Assert; import org.junit.Assume; @@ -33,7 +34,6 @@ import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DisplayColumn; import org.labkey.api.data.JdbcType; @@ -394,7 +394,7 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio SchemaKey schemaKey = null == schemaString ? new SchemaKey(null,"core") : SchemaKey.decode(schemaString); { // spring binding doesn't handle this form very well ( - if (!StringUtils.contains(getViewContext().getRequest().getContentType(),"json")) + if (!Strings.CS.contains(getViewContext().getRequest().getContentType(),"json")) { // white space is broken for separators, so rebind just in case if (StringUtils.isNotEmpty((String)getProperty("sep"))) @@ -403,7 +403,7 @@ public Object execute(SqlForm form, BindException errors) throws ServletExceptio form.setEol((String)getProperty("eol")); for (PropertyValue pv : getPropertyValues().getPropertyValues()) { - if (StringUtils.startsWith(pv.getName(),"parameters.")) + if (Strings.CS.startsWith(pv.getName(),"parameters.")) { String name = pv.getName().substring("parameters.".length()); Object value = pv.getValue(); From 2c9ca029cc22b86c8840f0a8a354f3344c848696 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 12 May 2026 16:37:52 -0700 Subject: [PATCH 10/10] execute2->execute --- query/src/org/labkey/query/controllers/LabKeySql.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query/src/org/labkey/query/controllers/LabKeySql.md b/query/src/org/labkey/query/controllers/LabKeySql.md index c72e19ec8e8..abba147e566 100644 --- a/query/src/org/labkey/query/controllers/LabKeySql.md +++ b/query/src/org/labkey/query/controllers/LabKeySql.md @@ -342,12 +342,12 @@ When writing LabKey SQL queries that work with JSON columns: ### **Fetching Live Data for LLM Inspection** -The `sql-execute2.view` endpoint is for **the LLM to inspect live data while generating code** — do not emit it in generated Python, R, or other scripts (use the `labkey` client API there instead). Always include `LIMIT` to avoid fetching excess rows. +The `sql-execute.view` endpoint is for **the LLM to inspect live data while generating code** — do not emit it in generated Python, R, or other scripts (use the `labkey` client API there instead). Always include `LIMIT` to avoid fetching excess rows. Unlike the LabKey client APIs (`selectRows`, `executeSql`), this endpoint does not automatically resolve lookups to display values — it returns exactly what the SQL selects. Use LabKey SQL's dot-notation to traverse lookups explicitly when you need a human-readable value. For example, `SELECT CreatedBy FROM lists.MyList` returns a raw integer user ID; `SELECT CreatedBy.DisplayName FROM lists.MyList` returns the display name. ```bash curl -H "Authorization: Bearer " \ - "https:////sql-execute2.view?schemaName=&sql=SELECT+...+LIMIT+20&format=tsv" + "https:////sql-execute.view?schemaName=&sql=SELECT+...+LIMIT+20&format=tsv" ``` The response is RFC 4180 TSV: a header row of column names followed by one data row per result. Values are rendered via `DisplayColumn.getTsvFormattedValue()` — dates as ISO-8601, multi-value columns correctly serialized. \ No newline at end of file