diff --git a/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml b/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml index 84e285fbbb2e..ffdd9051a751 100644 --- a/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml +++ b/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml @@ -23,3 +23,5 @@ links: url: https://issues.apache.org/jira/browse/SOLR-18513 - name: SOLR-18514 url: https://issues.apache.org/jira/browse/SOLR-18514 + - name: SOLR-18516 + url: https://issues.apache.org/jira/browse/SOLR-18516 diff --git a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java index 2764ba4b9fb4..5f02b57ece20 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java @@ -37,6 +37,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; @@ -45,8 +46,23 @@ import org.apache.solr.handler.component.ShardRequest; /** A command line tool for indexing Solr logs in the out-of-the-box log format. */ +@SuppressWarnings("UnnecessarilyFullyQualified") +@picocli.CommandLine.Command( + name = "postlogs", + description = "Indexes Solr logs in the out-of-the-box log format.", + footerHeading = "%nExamples:%n", + footer = { + " # Index all logs found under a directory", + " bin/solr postlogs -c gettingstarted --rootdir /var/solr/logs --solr-url" + + " http://localhost:8983" + }) public class PostLogsTool extends ToolBase { + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option COLLECTION_NAME_OPTION = Option.builder("c") .longOpt("name") @@ -56,6 +72,11 @@ public class PostLogsTool extends ToolBase { .desc("Name of the collection.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option ROOT_DIR_OPTION = Option.builder() .longOpt("rootdir") @@ -68,6 +89,33 @@ public class PostLogsTool extends ToolBase { /** Parameters for the postlogs command, independent of the command line parser. */ record PostLogsParams(String url, String rootDir, String credentials) {} + // --- picocli fields --- + // The connection group is mandatory (multiplicity "1"), mirroring the commons-cli path's + // manual IllegalArgumentException when no connection target is given. + + @picocli.CommandLine.ArgGroup(exclusive = true, multiplicity = "1") + private ConnectionOptions connectionOptions; + + @picocli.CommandLine.Mixin private CredentialsOptions credentialsOptions; + + @picocli.CommandLine.Option( + names = {"-c", "--name"}, + required = true, + paramLabel = "NAME", + description = "Name of the collection.") + private String nameOpt; + + @picocli.CommandLine.Option( + names = "--rootdir", + required = true, + paramLabel = "DIRECTORY", + description = "All files found at or below the root directory will be indexed.") + private String rootDirOpt; + + public PostLogsTool() { + this(new DefaultToolRuntime()); + } + public PostLogsTool(ToolRuntime runtime) { super(runtime); } @@ -639,6 +687,19 @@ public static String[] getRequestPurposeNames(Integer reqPurpose) { @Override public int callTool() throws Exception { - throw new UnsupportedOperationException("This tool does not yet support PicoCli"); + String url = resolveSolrUrl(credentialsOptions.credentials) + "/solr/" + nameOpt; + PostLogsParams params = new PostLogsParams(url, rootDirOpt, credentialsOptions.credentials); + runCommand(params); + return 0; + } + + private String resolveSolrUrl(String credentials) throws Exception { + String solrUrlArg = connectionOptions.effectiveSolrUrl(); + if (solrUrlArg != null) { + return CLIUtils.normalizeSolrUrl(solrUrlArg); + } + return CLIUtils.solrUrlFromConnection( + CloudSolrClient.CloudSolrClientConnection.parse(connectionOptions.effectiveZkHost()), + credentials); } } diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java index afd1c50467a7..859cee11e21d 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java @@ -74,6 +74,7 @@ import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.util.Utils; import org.apache.solr.util.RTimer; import org.w3c.dom.Document; @@ -82,6 +83,23 @@ import org.xml.sax.SAXException; /** Supports post command in the bin/solr script. */ +@SuppressWarnings("UnnecessarilyFullyQualified") +@picocli.CommandLine.Command( + name = "post", + description = + "The Post command sends one or more files, directories, URLs or literal data to a" + + " collection for indexing.", + footerHeading = "%nExamples:%n", + footer = { + " # Index all JSON files into a collection", + " bin/solr post -c gettingstarted *.json", + "", + " # Index a tab-separated file, using the CSV handler's separator param", + " bin/solr post -c gettingstarted --params \"separator=%%09\" --type text/csv data.tsv", + "", + " # Crawl a website one layer deep and index the pages found", + " bin/solr post -c gettingstarted --mode web --recursive 1 https://solr.apache.org/" + }) public class PostTool extends ToolBase { public static final String DEFAULT_FILE_TYPES = @@ -96,6 +114,11 @@ public class PostTool extends ToolBase { private static final int MAX_WEB_DEPTH = 10; public static final String DEFAULT_CONTENT_TYPE = "application/json"; + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option COLLECTION_NAME_OPTION = Option.builder("c") .longOpt("name") @@ -105,18 +128,33 @@ public class PostTool extends ToolBase { .desc("Name of the collection.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option SKIP_COMMIT_OPTION = Option.builder() .longOpt("skip-commit") .desc("Do not 'commit', and thus changes won't be visible till a commit occurs.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option OPTIMIZE_OPTION = Option.builder("o") .longOpt("optimize") .desc("Issue an optimize at end of posting documents.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option MODE_OPTION = Option.builder() .longOpt("mode") @@ -126,6 +164,11 @@ public class PostTool extends ToolBase { "Specifies how to run: 'files' crawls local directories, 'web' crawls websites, 'args' processes positional arguments, and 'stdin' reads standard input stream. Default: files.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option RECURSIVE_OPTION = Option.builder("r") .longOpt("recursive") @@ -135,6 +178,11 @@ public class PostTool extends ToolBase { .desc("For web crawl, how deep to go. default: 1") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option DELAY_OPTION = Option.builder("d") .longOpt("delay") @@ -145,6 +193,11 @@ public class PostTool extends ToolBase { "If recursive then delay will be the wait time between posts. default: 10 for web, 0 for files") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option TYPE_OPTION = Option.builder("t") .longOpt("type") @@ -153,6 +206,11 @@ public class PostTool extends ToolBase { .desc("Specify a specific mimetype to use, such as application/json.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option FILE_TYPES_OPTION = Option.builder("ft") .longOpt("filetypes") @@ -161,6 +219,11 @@ public class PostTool extends ToolBase { .desc("default: " + DEFAULT_FILE_TYPES) .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option PARAMS_OPTION = Option.builder() .longOpt("params") @@ -169,6 +232,11 @@ public class PostTool extends ToolBase { .desc("Values must be URL-encoded; these pass through to Solr update request.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option FORMAT_OPTION = Option.builder() .longOpt("format") @@ -176,6 +244,11 @@ public class PostTool extends ToolBase { "sends application/json content as Solr commands to /update instead of /update/json/docs.") .get(); + /** + * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated + * field. + */ + @Deprecated private static final Option DRY_RUN_OPTION = Option.builder() .longOpt("dry-run") @@ -212,6 +285,98 @@ public class PostTool extends ToolBase { PostTool.PageFetcher pageFetcher = new PostTool.PageFetcher(); + // --- picocli fields --- + // Named distinctly from the commons-cli-era instance fields above (type, recursive, delay, + // fileTypes, params, commit, optimize, dryRun, args), which postDocuments() still populates from + // whichever parser ran. + + @picocli.CommandLine.ArgGroup(exclusive = true, multiplicity = "0..1") + private ConnectionOptions connectionOptions; + + @picocli.CommandLine.Mixin private CredentialsOptions credentialsOptions; + + @picocli.CommandLine.Option( + names = {"-c", "--name"}, + required = true, + paramLabel = "name", + description = "Name of the collection.") + private String nameOpt; + + @picocli.CommandLine.Option( + names = "--skip-commit", + description = "Do not 'commit', and thus changes won't be visible till a commit occurs.") + private boolean skipCommitOpt; + + @picocli.CommandLine.Option( + names = {"-o", "--optimize"}, + description = "Issue an optimize at end of posting documents.") + private boolean optimizeOpt; + + @picocli.CommandLine.Option( + names = "--mode", + defaultValue = DATA_MODE_FILES, + paramLabel = "mode", + description = + "Which mode the Post command is running in, 'files' crawls local directory, 'web'" + + " crawls website, 'args' processes input args, and 'stdin' reads a command from" + + " standard in. default: files.") + private String modeOpt; + + @picocli.CommandLine.Option( + names = {"-r", "--recursive"}, + defaultValue = "1", + paramLabel = "recursive", + description = "For web crawl, how deep to go. default: 1") + private int recursiveOpt; + + @picocli.CommandLine.Option( + names = {"-d", "--delay"}, + paramLabel = "delay", + description = + "If recursive then delay will be the wait time between posts. default: 10 for web, 0" + + " for files") + private Integer delayOpt; + + @picocli.CommandLine.Option( + names = {"-t", "--type"}, + paramLabel = "content-type", + description = "Specify a specific mimetype to use, such as application/json.") + private String typeOpt; + + @picocli.CommandLine.Option( + names = {"-ft", "--filetypes"}, + defaultValue = DEFAULT_FILE_TYPES, + paramLabel = "[,,...]", + description = "default: " + DEFAULT_FILE_TYPES) + private String fileTypesOpt; + + @picocli.CommandLine.Option( + names = "--params", + defaultValue = "", + paramLabel = "=[&=...]", + description = "Values must be URL-encoded; these pass through to Solr update request.") + private String paramsOpt; + + @picocli.CommandLine.Option( + names = "--format", + description = + "sends application/json content as Solr commands to /update instead of" + + " /update/json/docs.") + private boolean formatOpt; + + @picocli.CommandLine.Option( + names = "--dry-run", + description = + "Performs a dry run of the posting process without actually sending documents to" + + " Solr. Only works with files mode.") + private boolean dryRunOpt; + + @picocli.CommandLine.Parameters( + arity = "0..*", + paramLabel = "FILE", + description = "Files, directories, urls or literal data to post, depending on --mode.") + private String[] postArgs = new String[0]; + static { DATA_MODES.add(DATA_MODE_FILES); DATA_MODES.add(DATA_MODE_ARGS); @@ -280,6 +445,10 @@ record PostToolParams( CrawlOptions crawl, UpdateOptions update) {} + public PostTool() { + this(new DefaultToolRuntime()); + } + public PostTool(ToolRuntime runtime) { super(runtime); } @@ -1401,7 +1570,37 @@ protected Set getLinksFromWebPage(URL url, InputStream is, String type, URI @Override public int callTool() throws Exception { - throw new UnsupportedOperationException("This tool does not yet support PicoCli"); + String resolvedSolrUrl = resolveSolrUrl(credentialsOptions.credentials); + URI updateUrl = new URI(resolvedSolrUrl + "/solr/" + nameOpt + "/update"); + int defaultDelay = modeOpt.equals(DATA_MODE_WEB) ? DEFAULT_WEB_DELAY : 0; + + PostToolParams postParams = + new PostToolParams( + updateUrl, + modeOpt, + dryRunOpt, + credentialsOptions.credentials, + postArgs, + new ContentOptions(typeOpt, formatOpt ? FORMAT_SOLR : "", paramsOpt), + new CrawlOptions( + fileTypesOpt, delayOpt != null ? delayOpt : defaultDelay, recursiveOpt), + new UpdateOptions(!skipCommitOpt, optimizeOpt)); + postDocuments(postParams); + return 0; + } + + /** As under commons-cli, a missing connection option falls back to the default URL silently. */ + private String resolveSolrUrl(String credentials) throws Exception { + String solrUrlArg = (connectionOptions != null) ? connectionOptions.effectiveSolrUrl() : null; + if (solrUrlArg != null) { + return CLIUtils.normalizeSolrUrl(solrUrlArg); + } + String zkHostArg = (connectionOptions != null) ? connectionOptions.effectiveZkHost() : null; + if (zkHostArg != null) { + return CLIUtils.solrUrlFromConnection( + CloudSolrClient.CloudSolrClientConnection.parse(zkHostArg), credentials); + } + return CLIUtils.getDefaultSolrUrl(); } /** Utility class to hold the result form a page fetch */ diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index 57b22e79d3ea..d8b8e2196132 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -86,7 +86,9 @@ ClusterTool.class, ConfigTool.class, ExportTool.class, - HealthcheckTool.class + HealthcheckTool.class, + PostTool.class, + PostLogsTool.class }) public class SolrCLI implements CLIO, java.util.concurrent.Callable { diff --git a/solr/core/src/test/org/apache/solr/cli/PostLogsToolCliTest.java b/solr/core/src/test/org/apache/solr/cli/PostLogsToolCliTest.java new file mode 100644 index 000000000000..436594983983 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cli/PostLogsToolCliTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PostLogsToolCliTest extends SolrCloudTestCase { + private static final String COLLECTION = "postLogsToolColl"; + + /** Runs the tool. Overridden by the picocli variant of this test. */ + protected int runTool(String[] args, Class clazz) throws Exception { + return CLITestHelper.runTool(args, clazz); + } + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure(); + CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION, 1, 1); + } + + private String solrUrl() { + return cluster.getJettySolrRunner(0).getBaseUrl().toString(); + } + + private Path logDirWithTwoQueries() throws Exception { + Path dir = createTempDir(); + String record = + "2019-12-09 15:05:%02d.931 INFO (qtp2103763750-21) [c:logs4 s:shard1 r:core_node2 x:logs4_shard1_replica_n1] o.a.s.c.S.Request [logs4_shard1_replica_n1] path=/select params={q=*:*&wt=javabin} hits=1 status=0 QTime=8\n"; + Files.writeString( + dir.resolve("solr.log"), + String.format(Locale.ROOT, record, 11) + String.format(Locale.ROOT, record, 12), + StandardCharsets.UTF_8); + return dir; + } + + @Test + public void testPostsLogRecords() throws Exception { + String[] args = { + "postlogs", + "-c", + COLLECTION, + "--solr-url", + solrUrl(), + "--rootdir", + logDirWithTwoQueries().toString() + }; + assertEquals(0, runTool(args, PostLogsTool.class)); + + long found = + cluster.getSolrClient().query(COLLECTION, new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(2, found); + } + + @Test + public void testFailsWithoutAConnectionTarget() throws Exception { + String[] args = {"postlogs", "-c", COLLECTION, "--rootdir", logDirWithTwoQueries().toString()}; + assertNotEquals(0, runTool(args, PostLogsTool.class)); + } +} diff --git a/solr/core/src/test/org/apache/solr/cli/PostLogsToolPicocliTest.java b/solr/core/src/test/org/apache/solr/cli/PostLogsToolPicocliTest.java new file mode 100644 index 000000000000..39aa9da7c7ec --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cli/PostLogsToolPicocliTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import java.util.Arrays; +import picocli.CommandLine; + +/** + * Runs all {@link PostLogsToolCliTest} tests through the picocli invocation path. + * + *

All {@code @Test} methods are inherited; only the invocation strategy is overridden. + */ +public class PostLogsToolPicocliTest extends PostLogsToolCliTest { + + @Override + protected int runTool(String[] args, Class clazz) throws Exception { + // args[0] is the tool name used by commons-cli dispatch; strip it for picocli. + String[] toolArgs = Arrays.copyOfRange(args, 1, args.length); + ToolRuntime runtime = new CLITestHelper.TestingRuntime(false); + ToolBase tool = clazz.getDeclaredConstructor(ToolRuntime.class).newInstance(runtime); + return new CommandLine(tool) + .setDefaultValueProvider(new CliDefaultValueProvider()) + .execute(toolArgs); + } +} diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolPicocliTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolPicocliTest.java new file mode 100644 index 000000000000..1a0de20e2c7f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cli/PostToolPicocliTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import java.util.Arrays; +import picocli.CommandLine; + +/** + * Runs all {@link PostToolTest} tests through the picocli invocation path. + * + *

All {@code @Test} methods are inherited; only the invocation strategy is overridden. + */ +public class PostToolPicocliTest extends PostToolTest { + + @Override + protected int runTool(String[] args, Class clazz) throws Exception { + // args[0] is the tool name used by commons-cli dispatch; strip it for picocli. + String[] toolArgs = Arrays.copyOfRange(args, 1, args.length); + ToolRuntime runtime = new CLITestHelper.TestingRuntime(false); + ToolBase tool = clazz.getDeclaredConstructor(ToolRuntime.class).newInstance(runtime); + return new CommandLine(tool) + .setDefaultValueProvider(new CliDefaultValueProvider()) + .execute(toolArgs); + } +} diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java index 5390fbeafcf9..d3c7c0641a3d 100644 --- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java @@ -55,6 +55,11 @@ @SolrTestCaseJ4.SuppressSSL public class PostToolTest extends SolrCloudTestCase { + /** Runs the tool. Overridden by the picocli variant of this test. */ + protected int runTool(String[] args, Class clazz) throws Exception { + return CLITestHelper.runTool(args, clazz); + } + @BeforeClass public static void setupClusterWithSecurityEnabled() throws Exception { configureCluster(2) @@ -92,7 +97,7 @@ public void testBasicRun() throws Exception { SecurityJson.USER_PASS, jsonDoc.toString(), }; - assertEquals(0, CLITestHelper.runTool(args, PostTool.class)); + assertEquals(0, runTool(args, PostTool.class)); int numFound = 0; int expectedDocCount = 1; @@ -130,7 +135,7 @@ public void testRunWithCollectionParam() throws Exception { String[] args = { "post", "-c", collection, "--credentials", SecurityJson.USER_PASS, jsonDoc.toString(), }; - assertEquals(0, CLITestHelper.runTool(args, PostTool.class)); + assertEquals(0, runTool(args, PostTool.class)); int numFound = 0; int expectedDocCount = 1; @@ -171,12 +176,12 @@ public void testRunCsvWithCustomSeparatorParam() throws Exception { "--credentials", SecurityJson.USER_PASS, "--params", - "\"separator=%09&header=false&fieldnames=id,title_s\"", + "separator=%09&header=false&fieldnames=id,title_s", "--type", "text/csv", tsvDoc.toString(), }; - assertEquals(0, CLITestHelper.runTool(args, PostTool.class)); + assertEquals(0, runTool(args, PostTool.class)); int numFound = 0; int expectedDocCount = 1; diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 50efa69ddbfa..dedb2783018c 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -367,6 +367,8 @@ task integrationTestsPicocli(type: BatsTask) { 'test/test_modules.bats', 'test/test_security_manager.bats', 'test/test_healthcheck.bats', + 'test/test_postlogs.bats', + 'test/test_extraction.bats', ] } diff --git a/solr/packaging/test/test_postlogs.bats b/solr/packaging/test/test_postlogs.bats index 76893566d43c..b54be5282385 100644 --- a/solr/packaging/test/test_postlogs.bats +++ b/solr/packaging/test/test_postlogs.bats @@ -42,7 +42,7 @@ teardown() { run solr create -c COLL_NAME assert_output --partial "Created collection 'COLL_NAME'" - run solr postlogs --solr-url http://localhost:${SOLR_PORT} -c COLL_NAME -rootdir ${SOLR_LOGS_DIR}/solr.log + run solr postlogs --solr-url http://localhost:${SOLR_PORT} -c COLL_NAME --rootdir ${SOLR_LOGS_DIR}/solr.log assert_output --partial 'Sending last batch' assert_output --partial 'Committed' diff --git a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc index 8336adf0a112..25397dcfe32a 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc @@ -29,6 +29,8 @@ *** xref:cli/solr-delete.adoc[delete] *** xref:cli/solr-export.adoc[export] *** xref:cli/solr-healthcheck.adoc[healthcheck] +*** xref:cli/solr-post.adoc[post] +*** xref:cli/solr-postlogs.adoc[postlogs] *** xref:cli/solr-start.adoc[start] *** xref:cli/solr-status.adoc[status] *** xref:cli/solr-stop.adoc[stop] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc index 4bf311d33155..c542dbb975bc 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc @@ -68,6 +68,8 @@ Any command not listed here is unavailable when the picocli implementation is en |xref:cli/solr-delete.adoc[bin/solr delete] |xref:cli/solr-export.adoc[bin/solr export] |xref:cli/solr-healthcheck.adoc[bin/solr healthcheck] +|xref:cli/solr-post.adoc[bin/solr post] +|xref:cli/solr-postlogs.adoc[bin/solr postlogs] |xref:cli/solr-start.adoc[bin/solr start] |xref:cli/solr-status.adoc[bin/solr status] |xref:cli/solr-stop.adoc[bin/solr stop] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-post.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-post.adoc new file mode 100644 index 000000000000..df98377f08fc --- /dev/null +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-post.adoc @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// DO NOT EDIT -- this page is auto-generated from picocli annotations. +// To update: modify the @Command/@Option annotations in the Java source, then run: +// ./gradlew :solr:solr-ref-guide:generateCliDocs + += bin/solr post +:page-toclevels: 2 + +[WARNING] +==== +This page documents the *experimental* picocli implementation of `bin/solr`, which is not enabled by default. +For what `bin/solr` does by default, see xref:solr-control-script-reference.adoc[]. +See xref:cli/index.adoc[] to enable this implementation and for its known limitations. +==== + +// tag::picocli-generated-man-section-synopsis[] +== Synopsis + +.... +bin/solr post [-hov] [--dry-run] [--format] [--skip-commit] -c=name [-d=delay] + [-ft=[,,...]] [--mode=mode] [--params== + [&=...]] [-r=recursive] [-t=content-type] + [-u=] [-s= | --solr-url= | + -z=] [FILE...] +.... + +// end::picocli-generated-man-section-synopsis[] + +// tag::picocli-generated-man-section-description[] +== Description + +The Post command sends one or more files, directories, URLs or literal data to a collection for indexing. + +// end::picocli-generated-man-section-description[] + +// tag::picocli-generated-man-section-options[] +== Options + +*-c*, *--name*=_name_:: + *(required)* Name of the collection. + +*-d*, *--delay*=_delay_:: + If recursive then delay will be the wait time between posts. default: 10 for web, 0 for files + +*--dry-run*:: + Performs a dry run of the posting process without actually sending documents to Solr. Only works with files mode. + +*--format*:: + sends application/json content as Solr commands to /update instead of /update/json/docs. + +*-ft, --filetypes*=_[,,...]_:: + default: xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log + +*-h*, *--help*:: + Print this help message and exit. + +*--mode*=_mode_:: + Which mode the Post command is running in, 'files' crawls local directory, 'web' crawls website, 'args' processes input args, and 'stdin' reads a command from standard in. default: files. + +*-o*, *--optimize*:: + Issue an optimize at end of posting documents. + +*--params*=_=[&=...]_:: + Values must be URL-encoded; these pass through to Solr update request. + +*-r*, *--recursive*=_recursive_:: + For web crawl, how deep to go. default: 1 + +*-s*, *--solr-connection*=__:: + Zookeeper or HTTP(s) connection string; unnecessary if SOLR_CONNECTION is defined in solr.in.sh; otherwise, defaults to localhost:9983. + +*--skip-commit*:: + Do not 'commit', and thus changes won't be visible till a commit occurs. + +*--solr-url*=__:: + Base Solr URL, which can be used to determine the zk-host if that's not known. + +*-t*, *--type*=_content-type_:: + Specify a specific mimetype to use, such as application/json. + +*-u*, *--credentials*=__:: + Credentials in the format username:password. Example: --credentials solr:SolrRocks + +*-v*, *--verbose*:: + Enable verbose mode. + +*-z*, *--zk-host*=__:: + Zookeeper connection string; unnecessary if ZK_HOST is defined in solr.in.sh; otherwise, defaults to localhost:9983. + +// end::picocli-generated-man-section-options[] + +// tag::picocli-generated-man-section-arguments[] +== Arguments + +[_FILE_...]:: + Files, directories, urls or literal data to post, depending on --mode. + +// end::picocli-generated-man-section-arguments[] + +// tag::picocli-generated-man-section-footer[] +== Examples +[%hardbreaks] + # Index all JSON files into a collection + bin/solr post -c gettingstarted *.json + +[%hardbreaks] + # Index a tab-separated file, using the CSV handler's separator param + bin/solr post -c gettingstarted --params "separator=%09" --type text/csv data.tsv + +[%hardbreaks] + # Crawl a website one layer deep and index the pages found + bin/solr post -c gettingstarted --mode web --recursive 1 https://solr.apache.org/ + +// end::picocli-generated-man-section-footer[] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-postlogs.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-postlogs.adoc new file mode 100644 index 000000000000..61c4a1a94cd7 --- /dev/null +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-postlogs.adoc @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// DO NOT EDIT -- this page is auto-generated from picocli annotations. +// To update: modify the @Command/@Option annotations in the Java source, then run: +// ./gradlew :solr:solr-ref-guide:generateCliDocs + += bin/solr postlogs +:page-toclevels: 2 + +[WARNING] +==== +This page documents the *experimental* picocli implementation of `bin/solr`, which is not enabled by default. +For what `bin/solr` does by default, see xref:solr-control-script-reference.adoc[]. +See xref:cli/index.adoc[] to enable this implementation and for its known limitations. +==== + +// tag::picocli-generated-man-section-synopsis[] +== Synopsis + +.... +bin/solr postlogs [-hv] -c=NAME --rootdir=DIRECTORY [-u=] + (-s= | --solr-url= | -z=) +.... + +// end::picocli-generated-man-section-synopsis[] + +// tag::picocli-generated-man-section-description[] +== Description + +Indexes Solr logs in the out-of-the-box log format. + +// end::picocli-generated-man-section-description[] + +// tag::picocli-generated-man-section-options[] +== Options + +*-c*, *--name*=_NAME_:: + *(required)* Name of the collection. + +*-h*, *--help*:: + Print this help message and exit. + +*--rootdir*=_DIRECTORY_:: + *(required)* All files found at or below the root directory will be indexed. + +*-s*, *--solr-connection*=__:: + Zookeeper or HTTP(s) connection string; unnecessary if SOLR_CONNECTION is defined in solr.in.sh; otherwise, defaults to localhost:9983. + +*--solr-url*=__:: + Base Solr URL, which can be used to determine the zk-host if that's not known. + +*-u*, *--credentials*=__:: + Credentials in the format username:password. Example: --credentials solr:SolrRocks + +*-v*, *--verbose*:: + Enable verbose mode. + +*-z*, *--zk-host*=__:: + Zookeeper connection string; unnecessary if ZK_HOST is defined in solr.in.sh; otherwise, defaults to localhost:9983. + +// end::picocli-generated-man-section-options[] + +// tag::picocli-generated-man-section-footer[] +== Examples +[%hardbreaks] + # Index all logs found under a directory + bin/solr postlogs -c gettingstarted --rootdir /var/solr/logs --solr-url http://localhost:8983 + +// end::picocli-generated-man-section-footer[] diff --git a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc index 6983720f7541..fe8f11a3aa0b 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc @@ -46,7 +46,7 @@ The `postlogs` command reads in Solr's log format and indexes it in a Solr colle Address of the collection, example http://localhost:8983/solr/\{collectionName}/. + -`-rootdir `:: +`--rootdir `:: + [%autowidth,frame=none] |===