1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2019 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.Iterator;
43 import java.util.List;
44 import java.util.Random;
45 import java.util.concurrent.TimeUnit;
46 import java.util.concurrent.locks.ReentrantReadWriteLock;
47 import java.util.logging.Level;
48 import javax.swing.AbstractAction;
49 import org.apache.solr.client.solrj.SolrQuery;
50 import org.apache.solr.client.solrj.SolrRequest;
51 import org.apache.solr.client.solrj.SolrServerException;
52 import org.apache.solr.client.solrj.impl.HttpSolrServer;
53 import org.apache.solr.client.solrj.impl.XMLResponseParser;
54 import org.apache.solr.client.solrj.request.CoreAdminRequest;
55 import org.apache.solr.client.solrj.response.CoreAdminResponse;
56 import org.apache.solr.client.solrj.response.QueryResponse;
57 import org.apache.solr.client.solrj.response.TermsResponse;
58 import org.apache.solr.common.SolrDocument;
59 import org.apache.solr.common.SolrDocumentList;
60 import org.apache.solr.common.SolrException;
61 import org.apache.solr.common.SolrInputDocument;
62 import org.apache.solr.common.params.CoreAdminParams;
63 import org.apache.solr.common.util.NamedList;
64 import org.openide.modules.InstalledFileLocator;
65 import org.openide.modules.Places;
66 import org.openide.util.NbBundle;
67 import org.openide.windows.WindowManager;
80
86
91
92 ID {
93 @Override
94 public String toString() {
95 return "id"; //NON-NLS
96 }
97 },
98 IMAGE_ID {
99 @Override
100 public String toString() {
101 return "image_id"; //NON-NLS
102 }
103 },
104 // This is not stored or indexed. it is copied to text by the schema
105 CONTENT {
106 @Override
107 public String toString() {
108 return "content"; //NON-NLS
109 }
110 },
111 // String representation for regular expression searching
112 CONTENT_STR {
113 @Override
114 public String toString() {
115 return "content_str"; //NON-NLS
116 }
117 },
118 // default search field. Populated by schema
119 TEXT {
120 @Override
121 public String toString() {
122 return "text"; //NON-NLS
123 }
124 },
125 // no longer populated. Was used for regular expression searching.
126 // Should not be used.
127 CONTENT_WS {
128 @Override
129 public String toString() {
130 return "content_ws"; //NON-NLS
131 }
132 },
133 CONTENT_JA {
134 @Override
135 public String toString() {
136 return "content_ja"; //NON-NLS
137 }
138 },
139 LANGUAGE {
140 @Override
141 public String toString() {
142 return "language"; //NON-NLS
143 }
144 },
145 FILE_NAME {
146 @Override
147 public String toString() {
148 return "file_name"; //NON-NLS
149 }
150 },
151 // note that we no longer store or index this field
152 CTIME {
153 @Override
154 public String toString() {
155 return "ctime"; //NON-NLS
156 }
157 },
158 // note that we no longer store or index this field
159 ATIME {
160 @Override
161 public String toString() {
162 return "atime"; //NON-NLS
163 }
164 },
165 // note that we no longer store or index this field
166 MTIME {
167 @Override
168 public String toString() {
169 return "mtime"; //NON-NLS
170 }
171 },
172 // note that we no longer store or index this field
173 CRTIME {
174 @Override
175 public String toString() {
176 return "crtime"; //NON-NLS
177 }
178 },
179 NUM_CHUNKS {
180 @Override
181 public String toString() {
182 return "num_chunks"; //NON-NLS
183 }
184 },
185 CHUNK_SIZE {
186 @Override
187 public String toString() {
188 return "chunk_size"; //NON-NLS
189 }
190 },
196 TERMFREQ {
197 @Override
198 public String toString() {
199 return "termfreq"; //NON-NLS
200 }
201 }
202 };
203
204 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
205 //max content size we can send to Solr
208 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
209 @Deprecated
215 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
216 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
217 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
218 private static final String
KEY =
"jjk#09s";
//NON-NLS
219 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
220 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
221 static final int DEFAULT_SOLR_STOP_PORT = 34343;
224 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
225 private static final String
SOLR =
"solr";
227
229
232
233 // A reference to the locally running Solr instance.
235
236 // A reference to the Solr server we are currently connected to for the Case.
237 // This could be a local or remote server.
239
242
247
254
255 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
256 serverAction = new ServerAction();
257 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
259
261 if (!solrHome.toFile().exists()) {
262 try {
263 Files.createDirectory(solrHome);
264 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
265 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
266 } catch (IOException ex) {
267 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
268 }
269 }
270 currentCoreLock = new ReentrantReadWriteLock(true);
271
272 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
273 }
274
276
278 try {
280 } catch (NumberFormatException nfe) {
281 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
282 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
283 }
284 } else {
285 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
287 }
288
290 try {
292 } catch (NumberFormatException nfe) {
293 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
294 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
295 }
296 } else {
297 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
299 }
300 }
301
302 @Override
303 public void finalize() throws java.lang.Throwable {
304 stop();
305 super.finalize();
306 }
307
309 serverAction.addPropertyChangeListener(l);
310 }
311
312 int getCurrentSolrServerPort() {
314 }
315
316 int getCurrentSolrStopPort() {
318 }
319
324
325 InputStream stream;
326 OutputStream out;
327 volatile boolean doRun = true;
328
330 this.stream = stream;
331 try {
332 final String log = Places.getUserDirectory().getAbsolutePath()
333 + File.separator + "var" + File.separator + "log" //NON-NLS
334 + File.separator + "solr.log." + type; //NON-NLS
335 File outputFile = new File(log.concat(".0"));
336 File first = new File(log.concat(".1"));
337 File second = new File(log.concat(".2"));
338 if (second.exists()) {
339 second.delete();
340 }
341 if (first.exists()) {
342 first.renameTo(second);
343 }
344 if (outputFile.exists()) {
345 outputFile.renameTo(first);
346 } else {
347 outputFile.createNewFile();
348 }
349 out = new FileOutputStream(outputFile);
350
351 } catch (Exception ex) {
352 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
353 }
354 }
355
356 void stopRun() {
357 doRun = false;
358 }
359
360 @Override
362
363 try (InputStreamReader isr = new InputStreamReader(stream);
364 BufferedReader br = new BufferedReader(isr);
366 BufferedWriter bw = new BufferedWriter(osw);) {
367
368 String line = null;
369 while (doRun && (line = br.readLine()) != null) {
370 bw.write(line);
371 bw.newLine();
372 if (DEBUG) {
373 //flush buffers if dev version for debugging
374 bw.flush();
375 }
376 }
377 bw.flush();
378 } catch (IOException ex) {
379 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
380 }
381 }
382 }
383
395
396 List<String> commandLine = new ArrayList<>();
397 commandLine.add(javaPath);
398 commandLine.add(MAX_SOLR_MEM_MB_PAR);
399 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
400 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
401 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
402 commandLine.add("-jar"); //NON-NLS
403 commandLine.add("start.jar"); //NON-NLS
404
405 commandLine.addAll(solrArguments);
406
407 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
408 solrProcessBuilder.directory(solrFolder);
409
410 // Redirect stdout and stderr to files to prevent blocking.
411 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
412 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
413
414 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
415 solrProcessBuilder.redirectError(solrStderrPath.toFile());
416
417 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
418 Process process = solrProcessBuilder.start();
419 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
420 return process;
421 }
422
428 List<Long> getSolrPIDs() {
429 List<Long> pids = new ArrayList<>();
430
431 //NOTE: these needs to be in sync with process start string in start()
432 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
433
435 if (pidsArr != null) {
436 for (int i = 0; i < pidsArr.length; ++i) {
437 pids.add(pidsArr[i]);
438 }
439 }
440
441 return pids;
442 }
443
448 void killSolr() {
449 List<Long> solrPids = getSolrPIDs();
450 for (long pid : solrPids) {
451 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
452 PlatformUtil.killProcess(pid);
453 }
454 }
455
461 @NbBundle.Messages({
462 "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",})
463 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
464 if (isRunning()) {
465 // If a Solr server is running we stop it.
466 stop();
467 }
468
469 if (!isPortAvailable(currentSolrServerPort)) {
470 // There is something already listening on our port. Let's see if
471 // this is from an earlier run that didn't successfully shut down
472 // and if so kill it.
473 final List<Long> pids = this.getSolrPIDs();
474
475 // If the culprit listening on the port is not a Solr process
476 // we refuse to start.
477 if (pids.isEmpty()) {
478 throw new SolrServerNoPortException(currentSolrServerPort);
479 }
480
481 // Ok, we've tried to stop it above but there still appears to be
482 // a Solr process listening on our port so we forcefully kill it.
483 killSolr();
484
485 // If either of the ports are still in use after our attempt to kill
486 // previously running processes we give up and throw an exception.
487 if (!isPortAvailable(currentSolrServerPort)) {
488 throw new SolrServerNoPortException(currentSolrServerPort);
489 }
490 if (!isPortAvailable(currentSolrStopPort)) {
491 throw new SolrServerNoPortException(currentSolrStopPort);
492 }
493 }
494
495 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
496
497 if (isPortAvailable(currentSolrServerPort)) {
498 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
499 try {
501 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
502 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
503
504 // Wait for the Solr server to start and respond to a status request.
505 for (int numRetries = 0; numRetries < 6; numRetries++) {
506 if (isRunning()) {
507 final List<Long> pids = this.getSolrPIDs();
508 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
509 return;
510 }
511
512 // Local Solr server did not respond so we sleep for
513 // 5 seconds before trying again.
514 try {
515 TimeUnit.SECONDS.sleep(5);
516 } catch (InterruptedException ex) {
517 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
518 }
519 }
520
521 // If we get here the Solr server has not responded to connection
522 // attempts in a timely fashion.
523 logger.log(Level.WARNING, "Local Solr server failed to respond to status requests.");
524 WindowManager.getDefault().invokeWhenUIReady(new Runnable() {
525 @Override
526 public void run() {
527 MessageNotifyUtil.Notify.error(
528 NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
529 Bundle.Server_status_failed_msg());
530 }
531 });
532 } catch (SecurityException ex) {
533 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
534 throw new KeywordSearchModuleException(
535 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
536 } catch (IOException ex) {
537 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
538 throw new KeywordSearchModuleException(
539 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
540 }
541 }
542 }
543
549 static boolean isPortAvailable(int port) {
550 ServerSocket ss = null;
551 try {
552
553 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
554 if (ss.isBound()) {
555 ss.setReuseAddress(true);
556 ss.close();
557 return true;
558 }
559
560 } catch (IOException e) {
561 } finally {
562 if (ss != null) {
563 try {
564 ss.close();
565 } catch (IOException e) {
566 /*
567 * should not be thrown
568 */
569 }
570 }
571 }
572 return false;
573 }
574
580 void changeSolrServerPort(int port) {
581 currentSolrServerPort = port;
582 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
583 }
584
590 void changeSolrStopPort(int port) {
591 currentSolrStopPort = port;
592 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
593 }
594
600 synchronized void stop() {
601
602 try {
603 // Close any open core before stopping server
604 closeCore();
605 } catch (KeywordSearchModuleException e) {
606 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
607 }
608
609 try {
610 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
611
612 //try graceful shutdown
613 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
614
615 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
616 process.waitFor();
617
618 //if still running, forcefully stop it
619 if (curSolrProcess != null) {
620 curSolrProcess.destroy();
621 curSolrProcess = null;
622 }
623
624 } catch (IOException | InterruptedException ex) {
625 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
626 } finally {
627 //stop Solr stream -> log redirect threads
628 try {
629 if (errorRedirectThread != null) {
630 errorRedirectThread.stopRun();
631 errorRedirectThread = null;
632 }
633 } finally {
634 //if still running, kill it
635 killSolr();
636 }
637
638 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
639 }
640 }
641
649 synchronized boolean isRunning() throws KeywordSearchModuleException {
650 try {
651
652 if (isPortAvailable(currentSolrServerPort)) {
653 return false;
654 }
655
656 // making a status request here instead of just doing solrServer.ping(), because
657 // that doesn't work when there are no cores
658 //TODO handle timeout in cases when some other type of server on that port
659 connectToSolrServer(localSolrServer);
660
661 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
662 } catch (SolrServerException ex) {
663
664 Throwable cause = ex.getRootCause();
665
666 // TODO: check if SocketExceptions should actually happen (is
667 // probably caused by starting a connection as the server finishes
668 // shutting down)
669 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
670 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
671 return false;
672 } else {
673 throw new KeywordSearchModuleException(
674 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
675 }
676 } catch (SolrException ex) {
677 // Just log 404 errors for now...
678 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
679 return false;
680 } catch (IOException ex) {
681 throw new KeywordSearchModuleException(
682 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
683 }
684
685 return true;
686 }
687
688 /*
689 * ** Convenience methods for use while we only open one case at a time ***
690 */
700 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
701 currentCoreLock.writeLock().lock();
702 try {
703 currentCore =
openCore(theCase, index);
704
705 try {
706 // execute a test query. if it fails, an exception will be thrown
708 } catch (NoOpenCoreException ex) {
709 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
710 }
711
712 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
713 } finally {
714 currentCoreLock.writeLock().unlock();
715 }
716 }
717
723 boolean coreIsOpen() {
724 currentCoreLock.readLock().lock();
725 try {
726 return (null != currentCore);
727 } finally {
728 currentCoreLock.readLock().unlock();
729 }
730 }
731
732 Index getIndexInfo() throws NoOpenCoreException {
733 currentCoreLock.readLock().lock();
734 try {
735 if (null == currentCore) {
736 throw new NoOpenCoreException();
737 }
738 return currentCore.getIndexInfo();
739 } finally {
740 currentCoreLock.readLock().unlock();
741 }
742 }
743
744 void closeCore() throws KeywordSearchModuleException {
745 currentCoreLock.writeLock().lock();
746 try {
747 if (null != currentCore) {
748 currentCore.close();
749 currentCore = null;
750 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
751 }
752 } finally {
753 currentCoreLock.writeLock().unlock();
754 }
755 }
756
757 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
758 currentCoreLock.readLock().lock();
759 try {
760 if (null == currentCore) {
761 throw new NoOpenCoreException();
762 }
763 TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk");
764 currentCore.addDocument(doc);
765 HealthMonitor.submitTimingMetric(metric);
766 } finally {
767 currentCoreLock.readLock().unlock();
768 }
769 }
770
779 @NbBundle.Messages({
780 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
781 void deleteCore(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException {
782 try {
783 HttpSolrServer solrServer;
784 if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
785 Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
786 solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
787 } else {
789 solrServer = new HttpSolrServer("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); //NON-NLS
790 }
791 connectToSolrServer(solrServer);
792 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
793 if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
794 /*
795 * Send a core unload request to the Solr server, with the
796 * parameter set that request deleting the index and the
797 * instance directory (deleteInstanceDir = true). Note that this
798 * removes everything related to the core on the server (the
799 * index directory, the configuration files, etc.), but does not
800 * delete the actual Solr text index because it is currently
801 * stored in the case directory.
802 */
803 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
804 }
805 } catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
806 // We will get a RemoteSolrException with cause == null and detailsMessage
807 // == "Already closed" if the core is not loaded. This is not an error in this scenario.
808 if (!ex.getMessage().equals("Already closed")) { // NON-NLS
809 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
810 }
811 }
812 }
813
825 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
826
827 try {
830 } else {
832 currentSolrServer =
new HttpSolrServer(
"http://" + properties.
getHost() +
":" + properties.
getPort() +
"/solr");
//NON-NLS
833 }
835 connectToSolrServer(currentSolrServer);
837
838 } catch (SolrServerException | IOException ex) {
839 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
840 }
841
842 try {
843 File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
844 if (!dataDir.exists()) {
845 dataDir.mkdirs();
846 }
847
848 if (!this.isRunning()) {
849 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
850 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
851 }
852
853 String coreName = index.getIndexName();
855 /*
856 * The core either does not exist or it is not loaded. Make a
857 * request that will cause the core to be created if it does not
858 * exist or loaded if it already exists.
859 */
860
861 // In single user mode, if there is a core.properties file already,
862 // we've hit a solr bug. Compensate by deleting it.
864 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
865 if (corePropertiesFile.toFile().exists()) {
866 try {
867 corePropertiesFile.toFile().delete();
868 } catch (Exception ex) {
869 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
870 }
871 }
872 }
873
874 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
875 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
876 createCoreRequest.setCoreName(coreName);
877 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
878 createCoreRequest.setIsLoadOnStartup(false);
879 createCoreRequest.setIsTransient(true);
880 currentSolrServer.request(createCoreRequest);
881 }
882
884 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
885 }
886
887 return new Core(coreName, theCase.getCaseType(), index);
888
889 } catch (Exception ex) {
890 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
891 }
892 }
893
905
906 Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
907 if (serverFilePath.toFile().exists()) {
908 try {
909 List<String> lines = Files.readAllLines(serverFilePath);
910 if (lines.isEmpty()) {
911 logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
912 } else if (!lines.get(0).contains(",")) {
913 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
914 } else {
915 String[] parts = lines.get(0).split(",");
916 if (parts.length != 2) {
917 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
918 } else {
920 }
921 }
922 } catch (IOException ex) {
923 logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
924 }
925 }
926
927 // Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
931 }
932
945 public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
throws KeywordSearchModuleException {
946 // Look for the solr server list file
947 String serverListName = "solrServerList.txt";
948 Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
949 if (serverListPath.toFile().exists()) {
950
951 // Read the list of solr servers
952 List<String> lines;
953 try {
954 lines = Files.readAllLines(serverListPath);
955 } catch (IOException ex) {
956 throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
957 }
958
959 // Remove any lines that don't contain a comma (these are likely just whitespace)
960 for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
961 String line = iterator.next();
962 if (!line.contains(",")) {
963 // Remove the current element from the iterator and the list.
964 iterator.remove();
965 }
966 }
967 if (lines.isEmpty()) {
968 throw new KeywordSearchModuleException(serverListName + " had no valid server information");
969 }
970
971 // Choose which server to use
972 int rnd = new Random().nextInt(lines.size());
973 String[] parts = lines.get(rnd).split(",");
974 if (parts.length != 2) {
975 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
976 }
977
978 // Split it up just to do a sanity check on the data
979 String host = parts[0];
980 String port = parts[1];
981 if (host.isEmpty() || port.isEmpty()) {
982 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
983 }
984
985 // Write the server data to a file
986 Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
987 try {
988 caseDirectoryPath.toFile().mkdirs();
989 if (!caseDirectoryPath.toFile().exists()) {
990 throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
991 }
992 Files.write(serverFile, lines.get(rnd).getBytes());
993 } catch (IOException ex) {
994 throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
995 }
996 }
997 }
998
1003
1006
1010 }
1011
1019 }
1020
1028 }
1029 }
1030
1036 void commit() throws SolrServerException, NoOpenCoreException {
1037 currentCoreLock.readLock().lock();
1038 try {
1039 if (null == currentCore) {
1040 throw new NoOpenCoreException();
1041 }
1042 currentCore.commit();
1043 } finally {
1044 currentCoreLock.readLock().unlock();
1045 }
1046 }
1047
1048 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
1049 currentCoreLock.readLock().lock();
1050 try {
1051 if (null == currentCore) {
1052 throw new NoOpenCoreException();
1053 }
1054 return currentCore.request(request);
1055 } finally {
1056 currentCoreLock.readLock().unlock();
1057 }
1058 }
1059
1071 currentCoreLock.readLock().lock();
1072 try {
1073 if (null == currentCore) {
1074 throw new NoOpenCoreException();
1075 }
1076 try {
1077 return currentCore.queryNumIndexedFiles();
1078 } catch (SolrServerException | IOException ex) {
1079 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
1080 }
1081 } finally {
1082 currentCoreLock.readLock().unlock();
1083 }
1084 }
1085
1096 currentCoreLock.readLock().lock();
1097 try {
1098 if (null == currentCore) {
1099 throw new NoOpenCoreException();
1100 }
1101 try {
1102 return currentCore.queryNumIndexedChunks();
1103 } catch (SolrServerException | IOException ex) {
1104 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
1105 }
1106 } finally {
1107 currentCoreLock.readLock().unlock();
1108 }
1109 }
1110
1121 currentCoreLock.readLock().lock();
1122 try {
1123 if (null == currentCore) {
1124 throw new NoOpenCoreException();
1125 }
1126 try {
1127 return currentCore.queryNumIndexedDocuments();
1128 } catch (SolrServerException | IOException ex) {
1129 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
1130 }
1131 } finally {
1132 currentCoreLock.readLock().unlock();
1133 }
1134 }
1135
1146 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
1147 currentCoreLock.readLock().lock();
1148 try {
1149 if (null == currentCore) {
1150 throw new NoOpenCoreException();
1151 }
1152 try {
1153 return currentCore.queryIsIndexed(contentID);
1154 } catch (SolrServerException | IOException ex) {
1155 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
1156 }
1157
1158 } finally {
1159 currentCoreLock.readLock().unlock();
1160 }
1161 }
1162
1175 currentCoreLock.readLock().lock();
1176 try {
1177 if (null == currentCore) {
1178 throw new NoOpenCoreException();
1179 }
1180 try {
1181 return currentCore.queryNumFileChunks(fileID);
1182 } catch (SolrServerException | IOException ex) {
1183 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
1184 }
1185 } finally {
1186 currentCoreLock.readLock().unlock();
1187 }
1188 }
1189
1200 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1201 currentCoreLock.readLock().lock();
1202 try {
1203 if (null == currentCore) {
1204 throw new NoOpenCoreException();
1205 }
1206 try {
1207 return currentCore.query(sq);
1208 } catch (SolrServerException ex) {
1209 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1210 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1211 }
1212 } finally {
1213 currentCoreLock.readLock().unlock();
1214 }
1215 }
1216
1228 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1229 currentCoreLock.readLock().lock();
1230 try {
1231 if (null == currentCore) {
1232 throw new NoOpenCoreException();
1233 }
1234 try {
1235 return currentCore.query(sq, method);
1236 } catch (SolrServerException | IOException ex) {
1237 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1238 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1239 }
1240 } finally {
1241 currentCoreLock.readLock().unlock();
1242 }
1243 }
1244
1255 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1256 currentCoreLock.readLock().lock();
1257 try {
1258 if (null == currentCore) {
1259 throw new NoOpenCoreException();
1260 }
1261 try {
1262 return currentCore.queryTerms(sq);
1263 } catch (SolrServerException | IOException ex) {
1264 logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1265 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1266 }
1267 } finally {
1268 currentCoreLock.readLock().unlock();
1269 }
1270 }
1271
1279 void deleteDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException {
1280 try {
1281 currentCoreLock.writeLock().lock();
1282 if (null == currentCore) {
1283 throw new NoOpenCoreException();
1284 }
1285 currentCore.deleteDataSource(dataSourceId);
1286 currentCore.commit();
1287 } finally {
1288 currentCoreLock.writeLock().unlock();
1289 }
1290 }
1291
1302 currentCoreLock.readLock().lock();
1303 try {
1304 if (null == currentCore) {
1305 throw new NoOpenCoreException();
1306 }
1307 return currentCore.getSolrContent(content.getId(), 0);
1308 } finally {
1309 currentCoreLock.readLock().unlock();
1310 }
1311 }
1312
1325 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
1326 currentCoreLock.readLock().lock();
1327 try {
1328 if (null == currentCore) {
1329 throw new NoOpenCoreException();
1330 }
1331 return currentCore.getSolrContent(content.getId(), chunkID);
1332 } finally {
1333 currentCoreLock.readLock().unlock();
1334 }
1335 }
1336
1347 currentCoreLock.readLock().lock();
1348 try {
1349 if (null == currentCore) {
1350 throw new NoOpenCoreException();
1351 }
1352 return currentCore.getSolrContent(objectID, 0);
1353 } finally {
1354 currentCoreLock.readLock().unlock();
1355 }
1356 }
1357
1368 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1369 currentCoreLock.readLock().lock();
1370 try {
1371 if (null == currentCore) {
1372 throw new NoOpenCoreException();
1373 }
1374 return currentCore.getSolrContent(objectID, chunkID);
1375 } finally {
1376 currentCoreLock.readLock().unlock();
1377 }
1378 }
1379
1391 }
1392
1401 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1403 CoreAdminRequest statusRequest = new CoreAdminRequest();
1404 statusRequest.setCoreName(null);
1405 statusRequest.setAction(CoreAdminParams.CoreAdminAction.STATUS);
1406 statusRequest.setIndexInfoNeeded(false);
1407 statusRequest.process(solrServer);
1409 }
1410
1424 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1425 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1426 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1427 }
1428
1440 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1441 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1442 if (null != dataDirPath) {
1443 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1444 return indexDir.exists();
1445 } else {
1446 return false;
1447 }
1448 }
1449
1450 class Core {
1451
1452 // handle to the core in Solr
1453 private final String name;
1454
1456
1457 private final Index textIndex;
1458
1459 // the server to access a core needs to be built from a URL with the
1460 // core in it, and is only good for core-specific operations
1461 private final HttpSolrServer solrCore;
1462
1463 private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
1464
1465 private Core(String name,
CaseType caseType, Index index) {
1466 this.name = name;
1467 this.caseType = caseType;
1468 this.textIndex = index;
1469
1470 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1471
1472 //TODO test these settings
1473 // socket read timeout, make large enough so can index larger files
1474 solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
1475 //solrCore.setConnectionTimeout(1000);
1476 solrCore.setDefaultMaxConnectionsPerHost(32);
1477 solrCore.setMaxTotalConnections(32);
1478 solrCore.setFollowRedirects(false); // defaults to false
1479 // allowCompression defaults to false.
1480 // Server side must support gzip or deflate for this to have any effect.
1481 solrCore.setAllowCompression(true);
1482 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1483
1484 }
1485
1491 String getName() {
1492 return name;
1493 }
1494
1495 private Index getIndexInfo() {
1496 return this.textIndex;
1497 }
1498
1499 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1500 return solrCore.query(sq);
1501 }
1502
1503 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1504 try {
1505 return solrCore.request(request);
1506 } catch (IOException e) {
1507 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1508 throw new SolrServerException(
1509 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1510 }
1511
1512 }
1513
1514 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1515 return solrCore.query(sq, method);
1516 }
1517
1518 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1519 QueryResponse qres = solrCore.query(sq);
1520 return qres.getTermsResponse();
1521 }
1522
1523 private void commit() throws SolrServerException {
1524 try {
1525 //commit and block
1526 solrCore.commit(true, true);
1527 } catch (IOException e) {
1528 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1529 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1530 }
1531 }
1532
1533 private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
1534 String dataSourceId = Long.toString(dsObjId);
1535 String deleteQuery = "image_id:" + dataSourceId;
1536
1537 solrCore.deleteByQuery(deleteQuery);
1538 }
1539
1540 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1541 try {
1542 solrCore.add(doc);
1543 } catch (SolrServerException ex) {
1544 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1545 throw new KeywordSearchModuleException(
1546 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1547 } catch (IOException ex) {
1548 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1549 throw new KeywordSearchModuleException(
1550 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1551 }
1552 }
1553
1565 final SolrQuery q = new SolrQuery();
1566 q.setQuery("*:*");
1567 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1568 if (chunkID != 0) {
1569 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1570 }
1571 q.addFilterQuery(filterQuery);
1572 q.setFields(Schema.TEXT.toString());
1573 try {
1574 // Get the first result.
1575 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1576
1577 if (!solrDocuments.isEmpty()) {
1578 SolrDocument solrDocument = solrDocuments.get(0);
1579 if (solrDocument != null) {
1580 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1581 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1582 {
1583 return fieldValues.toArray(new String[0])[0];
1584 } else // The indexed text for files has 2 values, the file name and the file content.
1585 // We return the file content value.
1586 {
1587 return fieldValues.toArray(new String[0])[1];
1588 }
1589 }
1590 }
1591 } catch (SolrServerException ex) {
1592 logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
1593 return null;
1594 }
1595
1596 return null;
1597 }
1598
1599 synchronized void close() throws KeywordSearchModuleException {
1600 // We only unload cores for "single-user" cases.
1602 return;
1603 }
1604
1605 try {
1606 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1607 } catch (SolrServerException ex) {
1608 throw new KeywordSearchModuleException(
1609 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1610 } catch (IOException ex) {
1611 throw new KeywordSearchModuleException(
1612 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1613 }
1614 }
1615
1627 }
1628
1639 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1640 q.setRows(0);
1641 int numChunks = (int)
query(q).getResults().getNumFound();
1642 return numChunks;
1643 }
1644
1656 SolrQuery q = new SolrQuery("*:*");
1657 q.setRows(0);
1658 return (
int)
query(q).getResults().getNumFound();
1659 }
1660
1670 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1671 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1672 SolrQuery q = new SolrQuery("*:*");
1673 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1674 //q.setFields(Server.Schema.ID.toString());
1675 q.setRows(0);
1676 return (
int)
query(q).getResults().getNumFound() != 0;
1677 }
1678
1690 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1691 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1692 final SolrQuery q
1693 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1694 q.setRows(0);
1695 return (
int)
query(q).getResults().getNumFound();
1696 }
1697 }
1698
1699 class ServerAction extends AbstractAction {
1700
1701 private static final long serialVersionUID = 1L;
1702
1703 @Override
1704 public void actionPerformed(ActionEvent e) {
1705 logger.log(Level.INFO, e.paramString().trim());
1706 }
1707 }
1708
1712 class SolrServerNoPortException extends SocketException {
1713
1714 private static final long serialVersionUID = 1L;
1715
1719 private final int port;
1720
1721 SolrServerNoPortException(int port) {
1722 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1723 Server.PROPERTIES_CURRENT_SERVER_PORT));
1724 this.port = port;
1725 }
1726
1727 int getPortNumber() {
1728 return port;
1729 }
1730 }
1731 }
static synchronized String getConfigSetting(String moduleName, String settingName)
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static IndexingServerProperties getMultiUserServerProperties(String caseDirectory)
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
static TimingMetric getTimingMetric(String name)
static int getMaxSolrVMSize()
static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
void addServerActionListener(PropertyChangeListener l)
static synchronized boolean settingExists(String moduleName, String settingName)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static void submitTimingMetric(TimingMetric metric)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)