1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.Iterator;
43 import java.util.List;
44 import java.util.Random;
45 import java.util.concurrent.locks.ReentrantReadWriteLock;
46 import java.util.logging.Level;
47 import javax.swing.AbstractAction;
48 import org.apache.solr.client.solrj.SolrQuery;
49 import org.apache.solr.client.solrj.SolrRequest;
50 import org.apache.solr.client.solrj.SolrServerException;
51 import org.apache.solr.client.solrj.impl.HttpSolrServer;
52 import org.apache.solr.client.solrj.impl.XMLResponseParser;
53 import org.apache.solr.client.solrj.request.CoreAdminRequest;
54 import org.apache.solr.client.solrj.response.CoreAdminResponse;
55 import org.apache.solr.client.solrj.response.QueryResponse;
56 import org.apache.solr.client.solrj.response.TermsResponse;
57 import org.apache.solr.common.SolrDocument;
58 import org.apache.solr.common.SolrDocumentList;
59 import org.apache.solr.common.SolrException;
60 import org.apache.solr.common.SolrInputDocument;
61 import org.apache.solr.common.params.CoreAdminParams;
62 import org.apache.solr.common.util.NamedList;
63 import org.openide.modules.InstalledFileLocator;
64 import org.openide.modules.Places;
65 import org.openide.util.NbBundle;
77
83
88
89 ID {
90 @Override
91 public String toString() {
92 return "id"; //NON-NLS
93 }
94 },
95 IMAGE_ID {
96 @Override
97 public String toString() {
98 return "image_id"; //NON-NLS
99 }
100 },
101 // This is not stored or index . it is copied to Text and Content_Ws
102 CONTENT {
103 @Override
104 public String toString() {
105 return "content"; //NON-NLS
106 }
107 },
108 CONTENT_STR {
109 @Override
110 public String toString() {
111 return "content_str"; //NON-NLS
112 }
113 },
114 TEXT {
115 @Override
116 public String toString() {
117 return "text"; //NON-NLS
118 }
119 },
120 CONTENT_WS {
121 @Override
122 public String toString() {
123 return "content_ws"; //NON-NLS
124 }
125 },
126 FILE_NAME {
127 @Override
128 public String toString() {
129 return "file_name"; //NON-NLS
130 }
131 },
132 // note that we no longer index this field
133 CTIME {
134 @Override
135 public String toString() {
136 return "ctime"; //NON-NLS
137 }
138 },
139 // note that we no longer index this field
140 ATIME {
141 @Override
142 public String toString() {
143 return "atime"; //NON-NLS
144 }
145 },
146 // note that we no longer index this field
147 MTIME {
148 @Override
149 public String toString() {
150 return "mtime"; //NON-NLS
151 }
152 },
153 // note that we no longer index this field
154 CRTIME {
155 @Override
156 public String toString() {
157 return "crtime"; //NON-NLS
158 }
159 },
160 NUM_CHUNKS {
161 @Override
162 public String toString() {
163 return "num_chunks"; //NON-NLS
164 }
165 },
166 CHUNK_SIZE {
167 @Override
168 public String toString() {
169 return "chunk_size"; //NON-NLS
170 }
171 }
172 };
173
174 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
175 //max content size we can send to Solr
178 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
179 @Deprecated
185 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
186 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
187 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
188 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
189 private static final String
KEY =
"jjk#09s";
//NON-NLS
190 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
191 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
192 static final int DEFAULT_SOLR_STOP_PORT = 34343;
195 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
196 private static final String
SOLR =
"solr";
198
200
203
204 // A reference to the locally running Solr instance.
206
207 // A reference to the Solr server we are currently connected to for the Case.
208 // This could be a local or remote server.
210
213
218
225
226 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
227 serverAction = new ServerAction();
228 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
230
232 if (!solrHome.toFile().exists()) {
233 try {
234 Files.createDirectory(solrHome);
235 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
236 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
237 } catch (IOException ex) {
238 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
239 }
240 }
241 currentCoreLock = new ReentrantReadWriteLock(true);
242
243 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
244 }
245
247
249 try {
251 } catch (NumberFormatException nfe) {
252 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
253 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
254 }
255 } else {
256 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
258 }
259
261 try {
263 } catch (NumberFormatException nfe) {
264 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
265 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
266 }
267 } else {
268 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
270 }
271 }
272
273 @Override
274 public void finalize() throws java.lang.Throwable {
275 stop();
276 super.finalize();
277 }
278
280 serverAction.addPropertyChangeListener(l);
281 }
282
283 int getCurrentSolrServerPort() {
285 }
286
287 int getCurrentSolrStopPort() {
289 }
290
295
296 InputStream stream;
297 OutputStream out;
298 volatile boolean doRun = true;
299
301 this.stream = stream;
302 try {
303 final String log = Places.getUserDirectory().getAbsolutePath()
304 + File.separator + "var" + File.separator + "log" //NON-NLS
305 + File.separator + "solr.log." + type; //NON-NLS
306 File outputFile = new File(log.concat(".0"));
307 File first = new File(log.concat(".1"));
308 File second = new File(log.concat(".2"));
309 if (second.exists()) {
310 second.delete();
311 }
312 if (first.exists()) {
313 first.renameTo(second);
314 }
315 if (outputFile.exists()) {
316 outputFile.renameTo(first);
317 } else {
318 outputFile.createNewFile();
319 }
320 out = new FileOutputStream(outputFile);
321
322 } catch (Exception ex) {
323 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
324 }
325 }
326
327 void stopRun() {
328 doRun = false;
329 }
330
331 @Override
333
334 try (InputStreamReader isr = new InputStreamReader(stream);
335 BufferedReader br = new BufferedReader(isr);
337 BufferedWriter bw = new BufferedWriter(osw);) {
338
339 String line = null;
340 while (doRun && (line = br.readLine()) != null) {
341 bw.write(line);
342 bw.newLine();
343 if (DEBUG) {
344 //flush buffers if dev version for debugging
345 bw.flush();
346 }
347 }
348 bw.flush();
349 } catch (IOException ex) {
350 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
351 }
352 }
353 }
354
365 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
366 List<String> commandLine = new ArrayList<>();
367 commandLine.add(javaPath);
368 commandLine.add(MAX_SOLR_MEM_MB_PAR);
369 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
370 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
371 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
372 commandLine.add("-jar"); //NON-NLS
373 commandLine.add("start.jar"); //NON-NLS
374
375 commandLine.addAll(solrArguments);
376
377 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
378 solrProcessBuilder.directory(solrFolder);
379
380 // Redirect stdout and stderr to files to prevent blocking.
381 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
382 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
383
384 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
385 solrProcessBuilder.redirectError(solrStderrPath.toFile());
386
387 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
388 Process process = solrProcessBuilder.start();
389 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
390 return process;
391 }
392
398 List<Long> getSolrPIDs() {
399 List<Long> pids = new ArrayList<>();
400
401 //NOTE: these needs to be in sync with process start string in start()
402 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
403
405 if (pidsArr != null) {
406 for (int i = 0; i < pidsArr.length; ++i) {
407 pids.add(pidsArr[i]);
408 }
409 }
410
411 return pids;
412 }
413
418 void killSolr() {
419 List<Long> solrPids = getSolrPIDs();
420 for (long pid : solrPids) {
421 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
422 PlatformUtil.killProcess(pid);
423 }
424 }
425
431 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
432 if (isRunning()) {
433 // If a Solr server is running we stop it.
434 stop();
435 }
436
437 if (!isPortAvailable(currentSolrServerPort)) {
438 // There is something already listening on our port. Let's see if
439 // this is from an earlier run that didn't successfully shut down
440 // and if so kill it.
441 final List<Long> pids = this.getSolrPIDs();
442
443 // If the culprit listening on the port is not a Solr process
444 // we refuse to start.
445 if (pids.isEmpty()) {
446 throw new SolrServerNoPortException(currentSolrServerPort);
447 }
448
449 // Ok, we've tried to stop it above but there still appears to be
450 // a Solr process listening on our port so we forcefully kill it.
451 killSolr();
452
453 // If either of the ports are still in use after our attempt to kill
454 // previously running processes we give up and throw an exception.
455 if (!isPortAvailable(currentSolrServerPort)) {
456 throw new SolrServerNoPortException(currentSolrServerPort);
457 }
458 if (!isPortAvailable(currentSolrStopPort)) {
459 throw new SolrServerNoPortException(currentSolrStopPort);
460 }
461 }
462
463 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
464
465 if (isPortAvailable(currentSolrServerPort)) {
466 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
467 try {
469 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
470 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
471
472 try {
473 //block for 10 seconds, give time to fully start the process
474 //so if it's restarted solr operations can be resumed seamlessly
475 Thread.sleep(10 * 1000);
476 } catch (InterruptedException ex) {
477 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
478 }
479
480 final List<Long> pids = this.getSolrPIDs();
481 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
482 } catch (SecurityException ex) {
483 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
484 throw new KeywordSearchModuleException(
485 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
486 } catch (IOException ex) {
487 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
488 throw new KeywordSearchModuleException(
489 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
490 }
491 }
492 }
493
499 static boolean isPortAvailable(int port) {
500 ServerSocket ss = null;
501 try {
502
503 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
504 if (ss.isBound()) {
505 ss.setReuseAddress(true);
506 ss.close();
507 return true;
508 }
509
510 } catch (IOException e) {
511 } finally {
512 if (ss != null) {
513 try {
514 ss.close();
515 } catch (IOException e) {
516 /*
517 * should not be thrown
518 */
519 }
520 }
521 }
522 return false;
523 }
524
530 void changeSolrServerPort(int port) {
531 currentSolrServerPort = port;
532 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
533 }
534
540 void changeSolrStopPort(int port) {
541 currentSolrStopPort = port;
542 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
543 }
544
550 synchronized void stop() {
551
552 try {
553 // Close any open core before stopping server
554 closeCore();
555 } catch (KeywordSearchModuleException e) {
556 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
557 }
558
559 try {
560 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
561
562 //try graceful shutdown
563 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
564
565 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
566 process.waitFor();
567
568 //if still running, forcefully stop it
569 if (curSolrProcess != null) {
570 curSolrProcess.destroy();
571 curSolrProcess = null;
572 }
573
574 } catch (IOException | InterruptedException ex) {
575 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
576 } finally {
577 //stop Solr stream -> log redirect threads
578 try {
579 if (errorRedirectThread != null) {
580 errorRedirectThread.stopRun();
581 errorRedirectThread = null;
582 }
583 } finally {
584 //if still running, kill it
585 killSolr();
586 }
587
588 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
589 }
590 }
591
599 synchronized boolean isRunning() throws KeywordSearchModuleException {
600 try {
601
602 if (isPortAvailable(currentSolrServerPort)) {
603 return false;
604 }
605
606 // making a status request here instead of just doing solrServer.ping(), because
607 // that doesn't work when there are no cores
608 //TODO handle timeout in cases when some other type of server on that port
609 connectToSolrServer(localSolrServer);
610
611 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
612 } catch (SolrServerException ex) {
613
614 Throwable cause = ex.getRootCause();
615
616 // TODO: check if SocketExceptions should actually happen (is
617 // probably caused by starting a connection as the server finishes
618 // shutting down)
619 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
620 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
621 return false;
622 } else {
623 throw new KeywordSearchModuleException(
624 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
625 }
626 } catch (SolrException ex) {
627 // Just log 404 errors for now...
628 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
629 return false;
630 } catch (IOException ex) {
631 throw new KeywordSearchModuleException(
632 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
633 }
634
635 return true;
636 }
637
638 /*
639 * ** Convenience methods for use while we only open one case at a time ***
640 */
650 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
651 currentCoreLock.writeLock().lock();
652 try {
653 currentCore =
openCore(theCase, index);
654
655 try {
656 // execute a test query. if it fails, an exception will be thrown
658 } catch (NoOpenCoreException ex) {
659 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
660 }
661
662 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
663 } finally {
664 currentCoreLock.writeLock().unlock();
665 }
666 }
667
673 boolean coreIsOpen() {
674 currentCoreLock.readLock().lock();
675 try {
676 return (null != currentCore);
677 } finally {
678 currentCoreLock.readLock().unlock();
679 }
680 }
681
682 Index getIndexInfo() throws NoOpenCoreException {
683 currentCoreLock.readLock().lock();
684 try {
685 if (null == currentCore) {
686 throw new NoOpenCoreException();
687 }
688 return currentCore.getIndexInfo();
689 } finally {
690 currentCoreLock.readLock().unlock();
691 }
692 }
693
694 void closeCore() throws KeywordSearchModuleException {
695 currentCoreLock.writeLock().lock();
696 try {
697 if (null != currentCore) {
698 currentCore.close();
699 currentCore = null;
700 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
701 }
702 } finally {
703 currentCoreLock.writeLock().unlock();
704 }
705 }
706
707 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
708 currentCoreLock.readLock().lock();
709 try {
710 if (null == currentCore) {
711 throw new NoOpenCoreException();
712 }
713 TimingMetric metric = EnterpriseHealthMonitor.getTimingMetric("Solr: Index chunk");
714 currentCore.addDocument(doc);
715 EnterpriseHealthMonitor.submitTimingMetric(metric);
716 } finally {
717 currentCoreLock.readLock().unlock();
718 }
719 }
720
729 @NbBundle.Messages({
730 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
731 void deleteCore(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException {
732 try {
733 HttpSolrServer solrServer;
734 if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
735 Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
736 solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
737 } else {
739 solrServer = new HttpSolrServer("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); //NON-NLS
740 }
741 connectToSolrServer(solrServer);
742 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
743 if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
744 /*
745 * Send a core unload request to the Solr server, with the
746 * parameter set that request deleting the index and the
747 * instance directory (deleteInstanceDir = true). Note that this
748 * removes everything related to the core on the server (the
749 * index directory, the configuration files, etc.), but does not
750 * delete the actual Solr text index because it is currently
751 * stored in the case directory.
752 */
753 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
754 }
755 } catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
756 // We will get a RemoteSolrException with cause == null and detailsMessage
757 // == "Already closed" if the core is not loaded. This is not an error in this scenario.
758 if (!ex.getMessage().equals("Already closed")) { // NON-NLS
759 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
760 }
761 }
762 }
763
775 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
776
777 try {
780 } else {
782 currentSolrServer =
new HttpSolrServer(
"http://" + properties.
getHost() +
":" + properties.
getPort() +
"/solr");
//NON-NLS
783 }
785 connectToSolrServer(currentSolrServer);
787
788 } catch (SolrServerException | IOException ex) {
789 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
790 }
791
792 try {
793 File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
794 if (!dataDir.exists()) {
795 dataDir.mkdirs();
796 }
797
798 if (!this.isRunning()) {
799 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
800 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
801 }
802
803 String coreName = index.getIndexName();
805 /*
806 * The core either does not exist or it is not loaded. Make a
807 * request that will cause the core to be created if it does not
808 * exist or loaded if it already exists.
809 */
810
811 // In single user mode, if there is a core.properties file already,
812 // we've hit a solr bug. Compensate by deleting it.
814 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
815 if (corePropertiesFile.toFile().exists()) {
816 try {
817 corePropertiesFile.toFile().delete();
818 } catch (Exception ex) {
819 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
820 }
821 }
822 }
823
824 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
825 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
826 createCoreRequest.setCoreName(coreName);
827 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
828 createCoreRequest.setIsLoadOnStartup(false);
829 createCoreRequest.setIsTransient(true);
830 currentSolrServer.request(createCoreRequest);
831 }
832
834 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
835 }
836
837 return new Core(coreName, theCase.getCaseType(), index);
838
839 } catch (Exception ex) {
840 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
841 }
842 }
843
853
854 Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
855 if(serverFilePath.toFile().exists()){
856 try{
857 List<String> lines = Files.readAllLines(serverFilePath);
858 if(lines.isEmpty()) {
859 logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
860 } else if (! lines.get(0).contains(",")) {
861 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
862 } else {
863 String[] parts = lines.get(0).split(",");
864 if(parts.length != 2) {
865 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
866 } else {
868 }
869 }
870 } catch (IOException ex) {
871 logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
872 }
873 }
874
875 // Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
879 }
880
894 public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
throws KeywordSearchModuleException {
895 // Look for the solr server list file
896 String serverListName = "solrServerList.txt";
897 Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
898 if(serverListPath.toFile().exists()){
899
900 // Read the list of solr servers
901 List<String> lines;
902 try{
903 lines = Files.readAllLines(serverListPath);
904 } catch (IOException ex){
905 throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
906 }
907
908 // Remove any lines that don't contain a comma (these are likely just whitespace)
909 for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
910 String line = iterator.next();
911 if (! line.contains(",")) {
912 // Remove the current element from the iterator and the list.
913 iterator.remove();
914 }
915 }
916 if(lines.isEmpty()) {
917 throw new KeywordSearchModuleException(serverListName + " had no valid server information");
918 }
919
920 // Choose which server to use
921 int rnd = new Random().nextInt(lines.size());
922 String[] parts = lines.get(rnd).split(",");
923 if(parts.length != 2) {
924 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
925 }
926
927 // Split it up just to do a sanity check on the data
928 String host = parts[0];
929 String port = parts[1];
930 if(host.isEmpty() || port.isEmpty()) {
931 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
932 }
933
934 // Write the server data to a file
935 Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
936 try {
937 caseDirectoryPath.toFile().mkdirs();
938 if (! caseDirectoryPath.toFile().exists()) {
939 throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
940 }
941 Files.write(serverFile, lines.get(rnd).getBytes());
942 } catch (IOException ex){
943 throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
944 }
945 }
946 }
947
954
958 }
959
966 }
967
974 }
975 }
976
982 void commit() throws SolrServerException, NoOpenCoreException {
983 currentCoreLock.readLock().lock();
984 try {
985 if (null == currentCore) {
986 throw new NoOpenCoreException();
987 }
988 currentCore.commit();
989 } finally {
990 currentCoreLock.readLock().unlock();
991 }
992 }
993
994 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
995 currentCoreLock.readLock().lock();
996 try {
997 if (null == currentCore) {
998 throw new NoOpenCoreException();
999 }
1000 return currentCore.request(request);
1001 } finally {
1002 currentCoreLock.readLock().unlock();
1003 }
1004 }
1005
1017 currentCoreLock.readLock().lock();
1018 try {
1019 if (null == currentCore) {
1020 throw new NoOpenCoreException();
1021 }
1022 try {
1023 return currentCore.queryNumIndexedFiles();
1024 } catch (SolrServerException | IOException ex) {
1025 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
1026 }
1027 } finally {
1028 currentCoreLock.readLock().unlock();
1029 }
1030 }
1031
1042 currentCoreLock.readLock().lock();
1043 try {
1044 if (null == currentCore) {
1045 throw new NoOpenCoreException();
1046 }
1047 try {
1048 return currentCore.queryNumIndexedChunks();
1049 } catch (SolrServerException | IOException ex) {
1050 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
1051 }
1052 } finally {
1053 currentCoreLock.readLock().unlock();
1054 }
1055 }
1056
1067 currentCoreLock.readLock().lock();
1068 try {
1069 if (null == currentCore) {
1070 throw new NoOpenCoreException();
1071 }
1072 try {
1073 return currentCore.queryNumIndexedDocuments();
1074 } catch (SolrServerException | IOException ex) {
1075 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
1076 }
1077 } finally {
1078 currentCoreLock.readLock().unlock();
1079 }
1080 }
1081
1092 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
1093 currentCoreLock.readLock().lock();
1094 try {
1095 if (null == currentCore) {
1096 throw new NoOpenCoreException();
1097 }
1098 try {
1099 return currentCore.queryIsIndexed(contentID);
1100 } catch (SolrServerException | IOException ex) {
1101 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
1102 }
1103
1104 } finally {
1105 currentCoreLock.readLock().unlock();
1106 }
1107 }
1108
1121 currentCoreLock.readLock().lock();
1122 try {
1123 if (null == currentCore) {
1124 throw new NoOpenCoreException();
1125 }
1126 try {
1127 return currentCore.queryNumFileChunks(fileID);
1128 } catch (SolrServerException | IOException ex) {
1129 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
1130 }
1131 } finally {
1132 currentCoreLock.readLock().unlock();
1133 }
1134 }
1135
1146 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1147 currentCoreLock.readLock().lock();
1148 try {
1149 if (null == currentCore) {
1150 throw new NoOpenCoreException();
1151 }
1152 try {
1153 return currentCore.query(sq);
1154 } catch (SolrServerException ex) {
1155 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1156 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1157 }
1158 } finally {
1159 currentCoreLock.readLock().unlock();
1160 }
1161 }
1162
1174 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1175 currentCoreLock.readLock().lock();
1176 try {
1177 if (null == currentCore) {
1178 throw new NoOpenCoreException();
1179 }
1180 try {
1181 return currentCore.query(sq, method);
1182 } catch (SolrServerException | IOException ex) {
1183 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1184 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1185 }
1186 } finally {
1187 currentCoreLock.readLock().unlock();
1188 }
1189 }
1190
1201 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1202 currentCoreLock.readLock().lock();
1203 try {
1204 if (null == currentCore) {
1205 throw new NoOpenCoreException();
1206 }
1207 try {
1208 return currentCore.queryTerms(sq);
1209 } catch (SolrServerException | IOException ex) {
1210 logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1211 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1212 }
1213 } finally {
1214 currentCoreLock.readLock().unlock();
1215 }
1216 }
1217
1228 currentCoreLock.readLock().lock();
1229 try {
1230 if (null == currentCore) {
1231 throw new NoOpenCoreException();
1232 }
1233 return currentCore.getSolrContent(content.getId(), 0);
1234 } finally {
1235 currentCoreLock.readLock().unlock();
1236 }
1237 }
1238
1251 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
1252 currentCoreLock.readLock().lock();
1253 try {
1254 if (null == currentCore) {
1255 throw new NoOpenCoreException();
1256 }
1257 return currentCore.getSolrContent(content.getId(), chunkID);
1258 } finally {
1259 currentCoreLock.readLock().unlock();
1260 }
1261 }
1262
1273 currentCoreLock.readLock().lock();
1274 try {
1275 if (null == currentCore) {
1276 throw new NoOpenCoreException();
1277 }
1278 return currentCore.getSolrContent(objectID, 0);
1279 } finally {
1280 currentCoreLock.readLock().unlock();
1281 }
1282 }
1283
1294 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1295 currentCoreLock.readLock().lock();
1296 try {
1297 if (null == currentCore) {
1298 throw new NoOpenCoreException();
1299 }
1300 return currentCore.getSolrContent(objectID, chunkID);
1301 } finally {
1302 currentCoreLock.readLock().unlock();
1303 }
1304 }
1305
1317 }
1318
1327 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1329 CoreAdminRequest statusRequest = new CoreAdminRequest();
1330 statusRequest.setCoreName( null );
1331 statusRequest.setAction( CoreAdminParams.CoreAdminAction.STATUS );
1332 statusRequest.setIndexInfoNeeded(false);
1333 statusRequest.process(solrServer);
1335 }
1336
1350 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1351 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1352 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1353 }
1354
1366 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1367 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1368 if (null != dataDirPath) {
1369 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1370 return indexDir.exists();
1371 } else {
1372 return false;
1373 }
1374 }
1375
1376 class Core {
1377
1378 // handle to the core in Solr
1379 private final String name;
1380
1382
1383 private final Index textIndex;
1384
1385 // the server to access a core needs to be built from a URL with the
1386 // core in it, and is only good for core-specific operations
1387 private final HttpSolrServer solrCore;
1388
1389 private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
1390
1391 private Core(String name,
CaseType caseType, Index index) {
1392 this.name = name;
1393 this.caseType = caseType;
1394 this.textIndex = index;
1395
1396 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1397
1398 //TODO test these settings
1399 // socket read timeout, make large enough so can index larger files
1400 solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
1401 //solrCore.setConnectionTimeout(1000);
1402 solrCore.setDefaultMaxConnectionsPerHost(32);
1403 solrCore.setMaxTotalConnections(32);
1404 solrCore.setFollowRedirects(false); // defaults to false
1405 // allowCompression defaults to false.
1406 // Server side must support gzip or deflate for this to have any effect.
1407 solrCore.setAllowCompression(true);
1408 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1409
1410 }
1411
1417 String getName() {
1418 return name;
1419 }
1420
1421 private Index getIndexInfo() {
1422 return this.textIndex;
1423 }
1424
1425 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1426 return solrCore.query(sq);
1427 }
1428
1429 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1430 try {
1431 return solrCore.request(request);
1432 } catch (IOException e) {
1433 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1434 throw new SolrServerException(
1435 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1436 }
1437
1438 }
1439
1440 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1441 return solrCore.query(sq, method);
1442 }
1443
1444 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1445 QueryResponse qres = solrCore.query(sq);
1446 return qres.getTermsResponse();
1447 }
1448
1449 private void commit() throws SolrServerException {
1450 try {
1451 //commit and block
1452 solrCore.commit(true, true);
1453 } catch (IOException e) {
1454 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1455 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1456 }
1457 }
1458
1459 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1460 try {
1461 solrCore.add(doc);
1462 } catch (SolrServerException ex) {
1463 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1464 throw new KeywordSearchModuleException(
1465 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1466 } catch (IOException ex) {
1467 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1468 throw new KeywordSearchModuleException(
1469 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1470 }
1471 }
1472
1483 final SolrQuery q = new SolrQuery();
1484 q.setQuery("*:*");
1485 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1486 if (chunkID != 0) {
1487 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1488 }
1489 q.addFilterQuery(filterQuery);
1490 q.setFields(Schema.TEXT.toString());
1491 try {
1492 // Get the first result.
1493 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1494
1495 if (!solrDocuments.isEmpty()) {
1496 SolrDocument solrDocument = solrDocuments.get(0);
1497 if (solrDocument != null) {
1498 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1499 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1500 {
1501 return fieldValues.toArray(new String[0])[0];
1502 } else // The indexed text for files has 2 values, the file name and the file content.
1503 // We return the file content value.
1504 {
1505 return fieldValues.toArray(new String[0])[1];
1506 }
1507 }
1508 }
1509 } catch (SolrServerException ex) {
1510 logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
1511 return null;
1512 }
1513
1514 return null;
1515 }
1516
1517 synchronized void close() throws KeywordSearchModuleException {
1518 // We only unload cores for "single-user" cases.
1520 return;
1521 }
1522
1523 try {
1524 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1525 } catch (SolrServerException ex) {
1526 throw new KeywordSearchModuleException(
1527 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1528 } catch (IOException ex) {
1529 throw new KeywordSearchModuleException(
1530 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1531 }
1532 }
1533
1545 }
1546
1557 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1558 q.setRows(0);
1559 int numChunks = (int)
query(q).getResults().getNumFound();
1560 return numChunks;
1561 }
1562
1574 SolrQuery q = new SolrQuery("*:*");
1575 q.setRows(0);
1576 return (
int)
query(q).getResults().getNumFound();
1577 }
1578
1588 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1589 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1590 SolrQuery q = new SolrQuery("*:*");
1591 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1592 //q.setFields(Server.Schema.ID.toString());
1593 q.setRows(0);
1594 return (
int)
query(q).getResults().getNumFound() != 0;
1595 }
1596
1608 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1609 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1610 final SolrQuery q
1611 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1612 q.setRows(0);
1613 return (
int)
query(q).getResults().getNumFound();
1614 }
1615 }
1616
1617 class ServerAction extends AbstractAction {
1618
1619 private static final long serialVersionUID = 1L;
1620
1621 @Override
1622 public void actionPerformed(ActionEvent e) {
1623 logger.log(Level.INFO, e.paramString().trim());
1624 }
1625 }
1626
1630 class SolrServerNoPortException extends SocketException {
1631
1632 private static final long serialVersionUID = 1L;
1633
1637 private final int port;
1638
1639 SolrServerNoPortException(int port) {
1640 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1641 Server.PROPERTIES_CURRENT_SERVER_PORT));
1642 this.port = port;
1643 }
1644
1645 int getPortNumber() {
1646 return port;
1647 }
1648 }
1649 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
static void submitTimingMetric(TimingMetric metric)
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static IndexingServerProperties getMultiUserServerProperties(String caseDirectory)
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static TimingMetric getTimingMetric(String name)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static final int MAX_SOLR_MEM_MB
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)