1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.List;
43 import java.util.concurrent.locks.ReentrantReadWriteLock;
44 import java.util.logging.Level;
45 import javax.swing.AbstractAction;
46 import org.apache.solr.client.solrj.SolrQuery;
47 import org.apache.solr.client.solrj.SolrRequest;
48 import org.apache.solr.client.solrj.SolrServerException;
49 import org.apache.solr.client.solrj.impl.HttpSolrServer;
50 import org.apache.solr.client.solrj.impl.XMLResponseParser;
51 import org.apache.solr.client.solrj.request.CoreAdminRequest;
52 import org.apache.solr.client.solrj.response.CoreAdminResponse;
53 import org.apache.solr.client.solrj.response.QueryResponse;
54 import org.apache.solr.client.solrj.response.TermsResponse;
55 import org.apache.solr.common.SolrDocument;
56 import org.apache.solr.common.SolrDocumentList;
57 import org.apache.solr.common.SolrException;
58 import org.apache.solr.common.SolrInputDocument;
59 import org.apache.solr.common.util.NamedList;
60 import org.openide.modules.InstalledFileLocator;
61 import org.openide.modules.Places;
62 import org.openide.util.NbBundle;
71
77
82
83 ID {
84 @Override
85 public String toString() {
86 return "id"; //NON-NLS
87 }
88 },
89 IMAGE_ID {
90 @Override
91 public String toString() {
92 return "image_id"; //NON-NLS
93 }
94 },
95 // This is not stored or index . it is copied to Text and Content_Ws
96 CONTENT {
97 @Override
98 public String toString() {
99 return "content"; //NON-NLS
100 }
101 },
102 CONTENT_STR {
103 @Override
104 public String toString() {
105 return "content_str"; //NON-NLS
106 }
107 },
108 TEXT {
109 @Override
110 public String toString() {
111 return "text"; //NON-NLS
112 }
113 },
114 CONTENT_WS {
115 @Override
116 public String toString() {
117 return "content_ws"; //NON-NLS
118 }
119 },
120 FILE_NAME {
121 @Override
122 public String toString() {
123 return "file_name"; //NON-NLS
124 }
125 },
126 // note that we no longer index this field
127 CTIME {
128 @Override
129 public String toString() {
130 return "ctime"; //NON-NLS
131 }
132 },
133 // note that we no longer index this field
134 ATIME {
135 @Override
136 public String toString() {
137 return "atime"; //NON-NLS
138 }
139 },
140 // note that we no longer index this field
141 MTIME {
142 @Override
143 public String toString() {
144 return "mtime"; //NON-NLS
145 }
146 },
147 // note that we no longer index this field
148 CRTIME {
149 @Override
150 public String toString() {
151 return "crtime"; //NON-NLS
152 }
153 },
154 NUM_CHUNKS {
155 @Override
156 public String toString() {
157 return "num_chunks"; //NON-NLS
158 }
159 },
160 CHUNK_SIZE {
161 @Override
162 public String toString() {
163 return "chunk_size"; //NON-NLS
164 }
165 }
166 };
167
168 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
169 //max content size we can send to Solr
172 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
173 @Deprecated
179 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
180 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
181 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
182 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
183 private static final String
KEY =
"jjk#09s";
//NON-NLS
184 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
185 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
186 static final int DEFAULT_SOLR_STOP_PORT = 34343;
189 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
190 private static final String
SOLR =
"solr";
192
194
197
198 // A reference to the locally running Solr instance.
200
201 // A reference to the Solr server we are currently connected to for the Case.
202 // This could be a local or remote server.
204
207
212
219
220 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
221 serverAction = new ServerAction();
222 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
224
226 if (!solrHome.toFile().exists()) {
227 try {
228 Files.createDirectory(solrHome);
229 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
230 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
231 } catch (IOException ex) {
232 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
233 }
234 }
235 currentCoreLock = new ReentrantReadWriteLock(true);
236
237 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
238 }
239
241
243 try {
245 } catch (NumberFormatException nfe) {
246 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
247 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
248 }
249 } else {
250 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
252 }
253
255 try {
257 } catch (NumberFormatException nfe) {
258 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
259 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
260 }
261 } else {
262 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
264 }
265 }
266
267 @Override
268 public void finalize() throws java.lang.Throwable {
269 stop();
270 super.finalize();
271 }
272
274 serverAction.addPropertyChangeListener(l);
275 }
276
277 int getCurrentSolrServerPort() {
279 }
280
281 int getCurrentSolrStopPort() {
283 }
284
289
290 InputStream stream;
291 OutputStream out;
292 volatile boolean doRun = true;
293
295 this.stream = stream;
296 try {
297 final String log = Places.getUserDirectory().getAbsolutePath()
298 + File.separator + "var" + File.separator + "log" //NON-NLS
299 + File.separator + "solr.log." + type; //NON-NLS
300 File outputFile = new File(log.concat(".0"));
301 File first = new File(log.concat(".1"));
302 File second = new File(log.concat(".2"));
303 if (second.exists()) {
304 second.delete();
305 }
306 if (first.exists()) {
307 first.renameTo(second);
308 }
309 if (outputFile.exists()) {
310 outputFile.renameTo(first);
311 } else {
312 outputFile.createNewFile();
313 }
314 out = new FileOutputStream(outputFile);
315
316 } catch (Exception ex) {
317 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
318 }
319 }
320
321 void stopRun() {
322 doRun = false;
323 }
324
325 @Override
327
328 try (InputStreamReader isr = new InputStreamReader(stream);
329 BufferedReader br = new BufferedReader(isr);
331 BufferedWriter bw = new BufferedWriter(osw);) {
332
333 String line = null;
334 while (doRun && (line = br.readLine()) != null) {
335 bw.write(line);
336 bw.newLine();
337 if (DEBUG) {
338 //flush buffers if dev version for debugging
339 bw.flush();
340 }
341 }
342 bw.flush();
343 } catch (IOException ex) {
344 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
345 }
346 }
347 }
348
359 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
360 List<String> commandLine = new ArrayList<>();
361 commandLine.add(javaPath);
362 commandLine.add(MAX_SOLR_MEM_MB_PAR);
363 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
364 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
365 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
366 commandLine.add("-jar"); //NON-NLS
367 commandLine.add("start.jar"); //NON-NLS
368
369 commandLine.addAll(solrArguments);
370
371 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
372 solrProcessBuilder.directory(solrFolder);
373
374 // Redirect stdout and stderr to files to prevent blocking.
375 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
376 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
377
378 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
379 solrProcessBuilder.redirectError(solrStderrPath.toFile());
380
381 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
382 Process process = solrProcessBuilder.start();
383 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
384 return process;
385 }
386
392 List<Long> getSolrPIDs() {
393 List<Long> pids = new ArrayList<>();
394
395 //NOTE: these needs to be in sync with process start string in start()
396 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
397
399 if (pidsArr != null) {
400 for (int i = 0; i < pidsArr.length; ++i) {
401 pids.add(pidsArr[i]);
402 }
403 }
404
405 return pids;
406 }
407
412 void killSolr() {
413 List<Long> solrPids = getSolrPIDs();
414 for (long pid : solrPids) {
415 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
416 PlatformUtil.killProcess(pid);
417 }
418 }
419
425 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
426 if (isRunning()) {
427 // If a Solr server is running we stop it.
428 stop();
429 }
430
431 if (!isPortAvailable(currentSolrServerPort)) {
432 // There is something already listening on our port. Let's see if
433 // this is from an earlier run that didn't successfully shut down
434 // and if so kill it.
435 final List<Long> pids = this.getSolrPIDs();
436
437 // If the culprit listening on the port is not a Solr process
438 // we refuse to start.
439 if (pids.isEmpty()) {
440 throw new SolrServerNoPortException(currentSolrServerPort);
441 }
442
443 // Ok, we've tried to stop it above but there still appears to be
444 // a Solr process listening on our port so we forcefully kill it.
445 killSolr();
446
447 // If either of the ports are still in use after our attempt to kill
448 // previously running processes we give up and throw an exception.
449 if (!isPortAvailable(currentSolrServerPort)) {
450 throw new SolrServerNoPortException(currentSolrServerPort);
451 }
452 if (!isPortAvailable(currentSolrStopPort)) {
453 throw new SolrServerNoPortException(currentSolrStopPort);
454 }
455 }
456
457 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
458
459 if (isPortAvailable(currentSolrServerPort)) {
460 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
461 try {
463 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
464 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
465
466 try {
467 //block for 10 seconds, give time to fully start the process
468 //so if it's restarted solr operations can be resumed seamlessly
469 Thread.sleep(10 * 1000);
470 } catch (InterruptedException ex) {
471 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
472 }
473
474 final List<Long> pids = this.getSolrPIDs();
475 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
476 } catch (SecurityException ex) {
477 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
478 throw new KeywordSearchModuleException(
479 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
480 } catch (IOException ex) {
481 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
482 throw new KeywordSearchModuleException(
483 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
484 }
485 }
486 }
487
493 static boolean isPortAvailable(int port) {
494 ServerSocket ss = null;
495 try {
496
497 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
498 if (ss.isBound()) {
499 ss.setReuseAddress(true);
500 ss.close();
501 return true;
502 }
503
504 } catch (IOException e) {
505 } finally {
506 if (ss != null) {
507 try {
508 ss.close();
509 } catch (IOException e) {
510 /*
511 * should not be thrown
512 */
513 }
514 }
515 }
516 return false;
517 }
518
524 void changeSolrServerPort(int port) {
525 currentSolrServerPort = port;
526 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
527 }
528
534 void changeSolrStopPort(int port) {
535 currentSolrStopPort = port;
536 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
537 }
538
544 synchronized void stop() {
545
546 try {
547 // Close any open core before stopping server
548 closeCore();
549 } catch (KeywordSearchModuleException e) {
550 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
551 }
552
553 try {
554 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
555
556 //try graceful shutdown
557 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
558
559 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
560 process.waitFor();
561
562 //if still running, forcefully stop it
563 if (curSolrProcess != null) {
564 curSolrProcess.destroy();
565 curSolrProcess = null;
566 }
567
568 } catch (IOException | InterruptedException ex) {
569 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
570 } finally {
571 //stop Solr stream -> log redirect threads
572 try {
573 if (errorRedirectThread != null) {
574 errorRedirectThread.stopRun();
575 errorRedirectThread = null;
576 }
577 } finally {
578 //if still running, kill it
579 killSolr();
580 }
581
582 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
583 }
584 }
585
593 synchronized boolean isRunning() throws KeywordSearchModuleException {
594 try {
595
596 if (isPortAvailable(currentSolrServerPort)) {
597 return false;
598 }
599
600 // making a status request here instead of just doing solrServer.ping(), because
601 // that doesn't work when there are no cores
602 //TODO handle timeout in cases when some other type of server on that port
603 CoreAdminRequest.getStatus(null, localSolrServer);
604
605 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
606 } catch (SolrServerException ex) {
607
608 Throwable cause = ex.getRootCause();
609
610 // TODO: check if SocketExceptions should actually happen (is
611 // probably caused by starting a connection as the server finishes
612 // shutting down)
613 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
614 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
615 return false;
616 } else {
617 throw new KeywordSearchModuleException(
618 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
619 }
620 } catch (SolrException ex) {
621 // Just log 404 errors for now...
622 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
623 return false;
624 } catch (IOException ex) {
625 throw new KeywordSearchModuleException(
626 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
627 }
628
629 return true;
630 }
631
632 /*
633 * ** Convenience methods for use while we only open one case at a time ***
634 */
644 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
645 currentCoreLock.writeLock().lock();
646 try {
647 currentCore =
openCore(theCase, index);
648
649 try {
650 // execute a test query. if it fails, an exception will be thrown
652 } catch (NoOpenCoreException ex) {
653 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
654 }
655
656 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
657 } finally {
658 currentCoreLock.writeLock().unlock();
659 }
660 }
661
667 boolean coreIsOpen() {
668 currentCoreLock.readLock().lock();
669 try {
670 return (null != currentCore);
671 } finally {
672 currentCoreLock.readLock().unlock();
673 }
674 }
675
676 Index getIndexInfo() throws NoOpenCoreException {
677 currentCoreLock.readLock().lock();
678 try {
679 if (null == currentCore) {
680 throw new NoOpenCoreException();
681 }
682 return currentCore.getIndexInfo();
683 } finally {
684 currentCoreLock.readLock().unlock();
685 }
686 }
687
688 void closeCore() throws KeywordSearchModuleException {
689 currentCoreLock.writeLock().lock();
690 try {
691 if (null != currentCore) {
692 currentCore.close();
693 currentCore = null;
694 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
695 }
696 } finally {
697 currentCoreLock.writeLock().unlock();
698 }
699 }
700
701 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
702 currentCoreLock.readLock().lock();
703 try {
704 if (null == currentCore) {
705 throw new NoOpenCoreException();
706 }
707 currentCore.addDocument(doc);
708 } finally {
709 currentCoreLock.readLock().unlock();
710 }
711 }
712
721 @NbBundle.Messages({
722 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
723 void deleteCore(String coreName, Case.CaseType caseType) throws KeywordSearchServiceException {
724 try {
725 HttpSolrServer solrServer;
726 if (caseType == CaseType.SINGLE_USER_CASE) {
727 Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
728 solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
729 } else {
730 String host = UserPreferences.getIndexingServerHost();
731 String port = UserPreferences.getIndexingServerPort();
732 solrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
733 }
734 connectToSolrServer(solrServer);
735 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
736 if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
737 /*
738 * Send a core unload request to the Solr server, with the
739 * parameter set that request deleting the index and the
740 * instance directory (deleteInstanceDir = true). Note that this
741 * removes everything related to the core on the server (the
742 * index directory, the configuration files, etc.), but does not
743 * delete the actual Solr text index because it is currently
744 * stored in the case directory.
745 */
746 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
747 }
748 } catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
749 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
750 }
751 }
752
764 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
765
766 try {
769 } else {
772 currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
773 }
774 connectToSolrServer(currentSolrServer);
775
776 } catch (SolrServerException | IOException ex) {
777 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
778 }
779
780 try {
781 File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
782 if (!dataDir.exists()) {
783 dataDir.mkdirs();
784 }
785
786 if (!this.isRunning()) {
787 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
788 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
789 }
790
791 String coreName = index.getIndexName();
793 /*
794 * The core either does not exist or it is not loaded. Make a
795 * request that will cause the core to be created if it does not
796 * exist or loaded if it already exists.
797 */
798
799 // In single user mode, if there is a core.properties file already,
800 // we've hit a solr bug. Compensate by deleting it.
802 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
803 if (corePropertiesFile.toFile().exists()) {
804 try {
805 corePropertiesFile.toFile().delete();
806 } catch (Exception ex) {
807 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
808 }
809 }
810 }
811
812 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
813 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
814 createCoreRequest.setCoreName(coreName);
815 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
816 createCoreRequest.setIsLoadOnStartup(false);
817 createCoreRequest.setIsTransient(true);
818 currentSolrServer.request(createCoreRequest);
819 }
820
822 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
823 }
824
825 return new Core(coreName, theCase.getCaseType(), index);
826
827 } catch (SolrServerException | SolrException | IOException ex) {
828 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
829 }
830 }
831
837 void commit() throws SolrServerException, NoOpenCoreException {
838 currentCoreLock.readLock().lock();
839 try {
840 if (null == currentCore) {
841 throw new NoOpenCoreException();
842 }
843 currentCore.commit();
844 } finally {
845 currentCoreLock.readLock().unlock();
846 }
847 }
848
849 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
850 currentCoreLock.readLock().lock();
851 try {
852 if (null == currentCore) {
853 throw new NoOpenCoreException();
854 }
855 return currentCore.request(request);
856 } finally {
857 currentCoreLock.readLock().unlock();
858 }
859 }
860
872 currentCoreLock.readLock().lock();
873 try {
874 if (null == currentCore) {
875 throw new NoOpenCoreException();
876 }
877 try {
878 return currentCore.queryNumIndexedFiles();
879 } catch (SolrServerException | IOException ex) {
880 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
881 }
882 } finally {
883 currentCoreLock.readLock().unlock();
884 }
885 }
886
897 currentCoreLock.readLock().lock();
898 try {
899 if (null == currentCore) {
900 throw new NoOpenCoreException();
901 }
902 try {
903 return currentCore.queryNumIndexedChunks();
904 } catch (SolrServerException | IOException ex) {
905 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
906 }
907 } finally {
908 currentCoreLock.readLock().unlock();
909 }
910 }
911
922 currentCoreLock.readLock().lock();
923 try {
924 if (null == currentCore) {
925 throw new NoOpenCoreException();
926 }
927 try {
928 return currentCore.queryNumIndexedDocuments();
929 } catch (SolrServerException | IOException ex) {
930 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
931 }
932 } finally {
933 currentCoreLock.readLock().unlock();
934 }
935 }
936
947 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
948 currentCoreLock.readLock().lock();
949 try {
950 if (null == currentCore) {
951 throw new NoOpenCoreException();
952 }
953 try {
954 return currentCore.queryIsIndexed(contentID);
955 } catch (SolrServerException | IOException ex) {
956 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
957 }
958
959 } finally {
960 currentCoreLock.readLock().unlock();
961 }
962 }
963
975 public int queryNumFileChunks(
long fileID)
throws KeywordSearchModuleException, NoOpenCoreException {
976 currentCoreLock.readLock().lock();
977 try {
978 if (null == currentCore) {
979 throw new NoOpenCoreException();
980 }
981 try {
982 return currentCore.queryNumFileChunks(fileID);
983 } catch (SolrServerException | IOException ex) {
984 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
985 }
986 } finally {
987 currentCoreLock.readLock().unlock();
988 }
989 }
990
1001 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1002 currentCoreLock.readLock().lock();
1003 try {
1004 if (null == currentCore) {
1005 throw new NoOpenCoreException();
1006 }
1007 try {
1008 return currentCore.query(sq);
1009 } catch (SolrServerException ex) {
1010 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1011 }
1012 } finally {
1013 currentCoreLock.readLock().unlock();
1014 }
1015 }
1016
1028 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1029 currentCoreLock.readLock().lock();
1030 try {
1031 if (null == currentCore) {
1032 throw new NoOpenCoreException();
1033 }
1034 try {
1035 return currentCore.query(sq, method);
1036 } catch (SolrServerException | IOException ex) {
1037 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1038 }
1039 } finally {
1040 currentCoreLock.readLock().unlock();
1041 }
1042 }
1043
1054 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1055 currentCoreLock.readLock().lock();
1056 try {
1057 if (null == currentCore) {
1058 throw new NoOpenCoreException();
1059 }
1060 try {
1061 return currentCore.queryTerms(sq);
1062 } catch (SolrServerException | IOException ex) {
1063 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1064 }
1065 } finally {
1066 currentCoreLock.readLock().unlock();
1067 }
1068 }
1069
1080 currentCoreLock.readLock().lock();
1081 try {
1082 if (null == currentCore) {
1083 throw new NoOpenCoreException();
1084 }
1085 return currentCore.getSolrContent(content.getId(), 0);
1086 } finally {
1087 currentCoreLock.readLock().unlock();
1088 }
1089 }
1090
1103 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
1104 currentCoreLock.readLock().lock();
1105 try {
1106 if (null == currentCore) {
1107 throw new NoOpenCoreException();
1108 }
1109 return currentCore.getSolrContent(content.getId(), chunkID);
1110 } finally {
1111 currentCoreLock.readLock().unlock();
1112 }
1113 }
1114
1125 currentCoreLock.readLock().lock();
1126 try {
1127 if (null == currentCore) {
1128 throw new NoOpenCoreException();
1129 }
1130 return currentCore.getSolrContent(objectID, 0);
1131 } finally {
1132 currentCoreLock.readLock().unlock();
1133 }
1134 }
1135
1146 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1147 currentCoreLock.readLock().lock();
1148 try {
1149 if (null == currentCore) {
1150 throw new NoOpenCoreException();
1151 }
1152 return currentCore.getSolrContent(objectID, chunkID);
1153 } finally {
1154 currentCoreLock.readLock().unlock();
1155 }
1156 }
1157
1169 }
1170
1179 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1180 CoreAdminRequest.getStatus(null, solrServer);
1181 }
1182
1196 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1197 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1198 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1199 }
1200
1212 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1213 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1214 if (null != dataDirPath) {
1215 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1216 return indexDir.exists();
1217 } else {
1218 return false;
1219 }
1220 }
1221
1222 class Core {
1223
1224 // handle to the core in Solr
1225 private final String name;
1226
1228
1229 private final Index textIndex;
1230
1231 // the server to access a core needs to be built from a URL with the
1232 // core in it, and is only good for core-specific operations
1233 private final HttpSolrServer solrCore;
1234
1235 private Core(String name,
CaseType caseType, Index index) {
1236 this.name = name;
1237 this.caseType = caseType;
1238 this.textIndex = index;
1239
1240 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1241
1242 //TODO test these settings
1243 //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
1244 //solrCore.setConnectionTimeout(1000);
1245 solrCore.setDefaultMaxConnectionsPerHost(2);
1246 solrCore.setMaxTotalConnections(5);
1247 solrCore.setFollowRedirects(false); // defaults to false
1248 // allowCompression defaults to false.
1249 // Server side must support gzip or deflate for this to have any effect.
1250 solrCore.setAllowCompression(true);
1251 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1252
1253 }
1254
1260 String getName() {
1261 return name;
1262 }
1263
1264 private Index getIndexInfo() {
1265 return this.textIndex;
1266 }
1267
1268 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1269 return solrCore.query(sq);
1270 }
1271
1272 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1273 try {
1274 return solrCore.request(request);
1275 } catch (IOException e) {
1276 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1277 throw new SolrServerException(
1278 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1279 }
1280
1281 }
1282
1283 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1284 return solrCore.query(sq, method);
1285 }
1286
1287 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1288 QueryResponse qres = solrCore.query(sq);
1289 return qres.getTermsResponse();
1290 }
1291
1292 private void commit() throws SolrServerException {
1293 try {
1294 //commit and block
1295 solrCore.commit(true, true);
1296 } catch (IOException e) {
1297 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1298 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1299 }
1300 }
1301
1302 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1303 try {
1304 solrCore.add(doc);
1305 } catch (SolrServerException ex) {
1306 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1307 throw new KeywordSearchModuleException(
1308 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1309 } catch (IOException ex) {
1310 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1311 throw new KeywordSearchModuleException(
1312 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1313 }
1314 }
1315
1325 final SolrQuery q = new SolrQuery();
1326 q.setQuery("*:*");
1327 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1328 if (chunkID != 0) {
1329 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1330 }
1331 q.addFilterQuery(filterQuery);
1332 q.setFields(Schema.TEXT.toString());
1333 try {
1334 // Get the first result.
1335 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1336
1337 if (!solrDocuments.isEmpty()) {
1338 SolrDocument solrDocument = solrDocuments.get(0);
1339 if (solrDocument != null) {
1340 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1341 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1342 {
1343 return fieldValues.toArray(new String[0])[0];
1344 } else // The indexed text for files has 2 values, the file name and the file content.
1345 // We return the file content value.
1346 {
1347 return fieldValues.toArray(new String[0])[1];
1348 }
1349 }
1350 }
1351 } catch (SolrServerException ex) {
1352 logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1353 return null;
1354 }
1355
1356 return null;
1357 }
1358
1359 synchronized void close() throws KeywordSearchModuleException {
1360 // We only unload cores for "single-user" cases.
1362 return;
1363 }
1364
1365 try {
1366 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1367 } catch (SolrServerException ex) {
1368 throw new KeywordSearchModuleException(
1369 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1370 } catch (IOException ex) {
1371 throw new KeywordSearchModuleException(
1372 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1373 }
1374 }
1375
1387 }
1388
1399 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1400 q.setRows(0);
1401 int numChunks = (int)
query(q).getResults().getNumFound();
1402 return numChunks;
1403 }
1404
1416 SolrQuery q = new SolrQuery("*:*");
1417 q.setRows(0);
1418 return (
int)
query(q).getResults().getNumFound();
1419 }
1420
1430 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1431 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1432 SolrQuery q = new SolrQuery("*:*");
1433 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1434 //q.setFields(Server.Schema.ID.toString());
1435 q.setRows(0);
1436 return (
int)
query(q).getResults().getNumFound() != 0;
1437 }
1438
1450 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1451 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1452 final SolrQuery q
1453 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1454 q.setRows(0);
1455 return (
int)
query(q).getResults().getNumFound();
1456 }
1457 }
1458
1459 class ServerAction extends AbstractAction {
1460
1461 private static final long serialVersionUID = 1L;
1462
1463 @Override
1464 public void actionPerformed(ActionEvent e) {
1465 logger.log(Level.INFO, e.paramString().trim());
1466 }
1467 }
1468
1472 class SolrServerNoPortException extends SocketException {
1473
1474 private static final long serialVersionUID = 1L;
1475
1479 private final int port;
1480
1481 SolrServerNoPortException(int port) {
1482 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1483 Server.PROPERTIES_CURRENT_SERVER_PORT));
1484 this.port = port;
1485 }
1486
1487 int getPortNumber() {
1488 return port;
1489 }
1490 }
1491 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static final int MAX_SOLR_MEM_MB
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)