1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.Iterator;
43 import java.util.List;
44 import java.util.Random;
45 import java.util.concurrent.locks.ReentrantReadWriteLock;
46 import java.util.logging.Level;
47 import javax.swing.AbstractAction;
48 import org.apache.solr.client.solrj.SolrQuery;
49 import org.apache.solr.client.solrj.SolrRequest;
50 import org.apache.solr.client.solrj.SolrServerException;
51 import org.apache.solr.client.solrj.impl.HttpSolrServer;
52 import org.apache.solr.client.solrj.impl.XMLResponseParser;
53 import org.apache.solr.client.solrj.request.CoreAdminRequest;
54 import org.apache.solr.client.solrj.response.CoreAdminResponse;
55 import org.apache.solr.client.solrj.response.QueryResponse;
56 import org.apache.solr.client.solrj.response.TermsResponse;
57 import org.apache.solr.common.SolrDocument;
58 import org.apache.solr.common.SolrDocumentList;
59 import org.apache.solr.common.SolrException;
60 import org.apache.solr.common.SolrInputDocument;
61 import org.apache.solr.common.params.CoreAdminParams;
62 import org.apache.solr.common.util.NamedList;
63 import org.openide.modules.InstalledFileLocator;
64 import org.openide.modules.Places;
65 import org.openide.util.NbBundle;
77
83
88
89 ID {
90 @Override
91 public String toString() {
92 return "id"; //NON-NLS
93 }
94 },
95 IMAGE_ID {
96 @Override
97 public String toString() {
98 return "image_id"; //NON-NLS
99 }
100 },
101 // This is not stored or index . it is copied to Text and Content_Ws
102 CONTENT {
103 @Override
104 public String toString() {
105 return "content"; //NON-NLS
106 }
107 },
108 CONTENT_STR {
109 @Override
110 public String toString() {
111 return "content_str"; //NON-NLS
112 }
113 },
114 TEXT {
115 @Override
116 public String toString() {
117 return "text"; //NON-NLS
118 }
119 },
120 CONTENT_WS {
121 @Override
122 public String toString() {
123 return "content_ws"; //NON-NLS
124 }
125 },
126 FILE_NAME {
127 @Override
128 public String toString() {
129 return "file_name"; //NON-NLS
130 }
131 },
132 // note that we no longer index this field
133 CTIME {
134 @Override
135 public String toString() {
136 return "ctime"; //NON-NLS
137 }
138 },
139 // note that we no longer index this field
140 ATIME {
141 @Override
142 public String toString() {
143 return "atime"; //NON-NLS
144 }
145 },
146 // note that we no longer index this field
147 MTIME {
148 @Override
149 public String toString() {
150 return "mtime"; //NON-NLS
151 }
152 },
153 // note that we no longer index this field
154 CRTIME {
155 @Override
156 public String toString() {
157 return "crtime"; //NON-NLS
158 }
159 },
160 NUM_CHUNKS {
161 @Override
162 public String toString() {
163 return "num_chunks"; //NON-NLS
164 }
165 },
166 CHUNK_SIZE {
167 @Override
168 public String toString() {
169 return "chunk_size"; //NON-NLS
170 }
171 }
172 };
173
174 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
175 //max content size we can send to Solr
178 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
179 @Deprecated
185 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
186 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
187 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
188 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
189 private static final String
KEY =
"jjk#09s";
//NON-NLS
190 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
191 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
192 static final int DEFAULT_SOLR_STOP_PORT = 34343;
195 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
196 private static final String
SOLR =
"solr";
198
200
203
204 // A reference to the locally running Solr instance.
206
207 // A reference to the Solr server we are currently connected to for the Case.
208 // This could be a local or remote server.
210
213
218
225
226 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
227 serverAction = new ServerAction();
228 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
230
232 if (!solrHome.toFile().exists()) {
233 try {
234 Files.createDirectory(solrHome);
235 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
236 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
237 } catch (IOException ex) {
238 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
239 }
240 }
241 currentCoreLock = new ReentrantReadWriteLock(true);
242
243 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
244 }
245
247
249 try {
251 } catch (NumberFormatException nfe) {
252 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
253 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
254 }
255 } else {
256 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
258 }
259
261 try {
263 } catch (NumberFormatException nfe) {
264 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
265 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
266 }
267 } else {
268 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
270 }
271 }
272
273 @Override
274 public void finalize() throws java.lang.Throwable {
275 stop();
276 super.finalize();
277 }
278
280 serverAction.addPropertyChangeListener(l);
281 }
282
283 int getCurrentSolrServerPort() {
285 }
286
287 int getCurrentSolrStopPort() {
289 }
290
295
296 InputStream stream;
297 OutputStream out;
298 volatile boolean doRun = true;
299
301 this.stream = stream;
302 try {
303 final String log = Places.getUserDirectory().getAbsolutePath()
304 + File.separator + "var" + File.separator + "log" //NON-NLS
305 + File.separator + "solr.log." + type; //NON-NLS
306 File outputFile = new File(log.concat(".0"));
307 File first = new File(log.concat(".1"));
308 File second = new File(log.concat(".2"));
309 if (second.exists()) {
310 second.delete();
311 }
312 if (first.exists()) {
313 first.renameTo(second);
314 }
315 if (outputFile.exists()) {
316 outputFile.renameTo(first);
317 } else {
318 outputFile.createNewFile();
319 }
320 out = new FileOutputStream(outputFile);
321
322 } catch (Exception ex) {
323 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
324 }
325 }
326
327 void stopRun() {
328 doRun = false;
329 }
330
331 @Override
333
334 try (InputStreamReader isr = new InputStreamReader(stream);
335 BufferedReader br = new BufferedReader(isr);
337 BufferedWriter bw = new BufferedWriter(osw);) {
338
339 String line = null;
340 while (doRun && (line = br.readLine()) != null) {
341 bw.write(line);
342 bw.newLine();
343 if (DEBUG) {
344 //flush buffers if dev version for debugging
345 bw.flush();
346 }
347 }
348 bw.flush();
349 } catch (IOException ex) {
350 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
351 }
352 }
353 }
354
365 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
366 List<String> commandLine = new ArrayList<>();
367 commandLine.add(javaPath);
368 commandLine.add(MAX_SOLR_MEM_MB_PAR);
369 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
370 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
371 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
372 commandLine.add("-jar"); //NON-NLS
373 commandLine.add("start.jar"); //NON-NLS
374
375 commandLine.addAll(solrArguments);
376
377 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
378 solrProcessBuilder.directory(solrFolder);
379
380 // Redirect stdout and stderr to files to prevent blocking.
381 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
382 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
383
384 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
385 solrProcessBuilder.redirectError(solrStderrPath.toFile());
386
387 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
388 Process process = solrProcessBuilder.start();
389 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
390 return process;
391 }
392
398 List<Long> getSolrPIDs() {
399 List<Long> pids = new ArrayList<>();
400
401 //NOTE: these needs to be in sync with process start string in start()
402 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
403
405 if (pidsArr != null) {
406 for (int i = 0; i < pidsArr.length; ++i) {
407 pids.add(pidsArr[i]);
408 }
409 }
410
411 return pids;
412 }
413
418 void killSolr() {
419 List<Long> solrPids = getSolrPIDs();
420 for (long pid : solrPids) {
421 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
422 PlatformUtil.killProcess(pid);
423 }
424 }
425
431 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
432 if (isRunning()) {
433 // If a Solr server is running we stop it.
434 stop();
435 }
436
437 if (!isPortAvailable(currentSolrServerPort)) {
438 // There is something already listening on our port. Let's see if
439 // this is from an earlier run that didn't successfully shut down
440 // and if so kill it.
441 final List<Long> pids = this.getSolrPIDs();
442
443 // If the culprit listening on the port is not a Solr process
444 // we refuse to start.
445 if (pids.isEmpty()) {
446 throw new SolrServerNoPortException(currentSolrServerPort);
447 }
448
449 // Ok, we've tried to stop it above but there still appears to be
450 // a Solr process listening on our port so we forcefully kill it.
451 killSolr();
452
453 // If either of the ports are still in use after our attempt to kill
454 // previously running processes we give up and throw an exception.
455 if (!isPortAvailable(currentSolrServerPort)) {
456 throw new SolrServerNoPortException(currentSolrServerPort);
457 }
458 if (!isPortAvailable(currentSolrStopPort)) {
459 throw new SolrServerNoPortException(currentSolrStopPort);
460 }
461 }
462
463 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
464
465 if (isPortAvailable(currentSolrServerPort)) {
466 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
467 try {
469 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
470 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
471
472 try {
473 //block for 10 seconds, give time to fully start the process
474 //so if it's restarted solr operations can be resumed seamlessly
475 Thread.sleep(10 * 1000);
476 } catch (InterruptedException ex) {
477 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
478 }
479
480 final List<Long> pids = this.getSolrPIDs();
481 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
482 } catch (SecurityException ex) {
483 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
484 throw new KeywordSearchModuleException(
485 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
486 } catch (IOException ex) {
487 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
488 throw new KeywordSearchModuleException(
489 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
490 }
491 }
492 }
493
499 static boolean isPortAvailable(int port) {
500 ServerSocket ss = null;
501 try {
502
503 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
504 if (ss.isBound()) {
505 ss.setReuseAddress(true);
506 ss.close();
507 return true;
508 }
509
510 } catch (IOException e) {
511 } finally {
512 if (ss != null) {
513 try {
514 ss.close();
515 } catch (IOException e) {
516 /*
517 * should not be thrown
518 */
519 }
520 }
521 }
522 return false;
523 }
524
530 void changeSolrServerPort(int port) {
531 currentSolrServerPort = port;
532 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
533 }
534
540 void changeSolrStopPort(int port) {
541 currentSolrStopPort = port;
542 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
543 }
544
550 synchronized void stop() {
551
552 try {
553 // Close any open core before stopping server
554 closeCore();
555 } catch (KeywordSearchModuleException e) {
556 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
557 }
558
559 try {
560 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
561
562 //try graceful shutdown
563 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
564
565 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
566 process.waitFor();
567
568 //if still running, forcefully stop it
569 if (curSolrProcess != null) {
570 curSolrProcess.destroy();
571 curSolrProcess = null;
572 }
573
574 } catch (IOException | InterruptedException ex) {
575 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
576 } finally {
577 //stop Solr stream -> log redirect threads
578 try {
579 if (errorRedirectThread != null) {
580 errorRedirectThread.stopRun();
581 errorRedirectThread = null;
582 }
583 } finally {
584 //if still running, kill it
585 killSolr();
586 }
587
588 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
589 }
590 }
591
599 synchronized boolean isRunning() throws KeywordSearchModuleException {
600 try {
601
602 if (isPortAvailable(currentSolrServerPort)) {
603 return false;
604 }
605
606 // making a status request here instead of just doing solrServer.ping(), because
607 // that doesn't work when there are no cores
608 //TODO handle timeout in cases when some other type of server on that port
609 connectToSolrServer(localSolrServer);
610
611 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
612 } catch (SolrServerException ex) {
613
614 Throwable cause = ex.getRootCause();
615
616 // TODO: check if SocketExceptions should actually happen (is
617 // probably caused by starting a connection as the server finishes
618 // shutting down)
619 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
620 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
621 return false;
622 } else {
623 throw new KeywordSearchModuleException(
624 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
625 }
626 } catch (SolrException ex) {
627 // Just log 404 errors for now...
628 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
629 return false;
630 } catch (IOException ex) {
631 throw new KeywordSearchModuleException(
632 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
633 }
634
635 return true;
636 }
637
638 /*
639 * ** Convenience methods for use while we only open one case at a time ***
640 */
650 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
651 currentCoreLock.writeLock().lock();
652 try {
653 currentCore =
openCore(theCase, index);
654
655 try {
656 // execute a test query. if it fails, an exception will be thrown
658 } catch (NoOpenCoreException ex) {
659 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
660 }
661
662 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
663 } finally {
664 currentCoreLock.writeLock().unlock();
665 }
666 }
667
673 boolean coreIsOpen() {
674 currentCoreLock.readLock().lock();
675 try {
676 return (null != currentCore);
677 } finally {
678 currentCoreLock.readLock().unlock();
679 }
680 }
681
682 Index getIndexInfo() throws NoOpenCoreException {
683 currentCoreLock.readLock().lock();
684 try {
685 if (null == currentCore) {
686 throw new NoOpenCoreException();
687 }
688 return currentCore.getIndexInfo();
689 } finally {
690 currentCoreLock.readLock().unlock();
691 }
692 }
693
694 void closeCore() throws KeywordSearchModuleException {
695 currentCoreLock.writeLock().lock();
696 try {
697 if (null != currentCore) {
698 currentCore.close();
699 currentCore = null;
700 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
701 }
702 } finally {
703 currentCoreLock.writeLock().unlock();
704 }
705 }
706
707 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
708 currentCoreLock.readLock().lock();
709 try {
710 if (null == currentCore) {
711 throw new NoOpenCoreException();
712 }
713 TimingMetric metric = EnterpriseHealthMonitor.getTimingMetric("Solr: Index chunk");
714 currentCore.addDocument(doc);
715 EnterpriseHealthMonitor.submitTimingMetric(metric);
716 } finally {
717 currentCoreLock.readLock().unlock();
718 }
719 }
720
729 @NbBundle.Messages({
730 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
731 void deleteCore(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException {
732 try {
733 HttpSolrServer solrServer;
734 if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
735 Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
736 solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
737 } else {
739 solrServer = new HttpSolrServer("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); //NON-NLS
740 }
741 connectToSolrServer(solrServer);
742 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
743 if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
744 /*
745 * Send a core unload request to the Solr server, with the
746 * parameter set that request deleting the index and the
747 * instance directory (deleteInstanceDir = true). Note that this
748 * removes everything related to the core on the server (the
749 * index directory, the configuration files, etc.), but does not
750 * delete the actual Solr text index because it is currently
751 * stored in the case directory.
752 */
753 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
754 }
755 } catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
756 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
757 }
758 }
759
771 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
772
773 try {
776 } else {
778 currentSolrServer =
new HttpSolrServer(
"http://" + properties.
getHost() +
":" + properties.
getPort() +
"/solr");
//NON-NLS
779 }
781 connectToSolrServer(currentSolrServer);
783
784 } catch (SolrServerException | IOException ex) {
785 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
786 }
787
788 try {
789 File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
790 if (!dataDir.exists()) {
791 dataDir.mkdirs();
792 }
793
794 if (!this.isRunning()) {
795 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
796 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
797 }
798
799 String coreName = index.getIndexName();
801 /*
802 * The core either does not exist or it is not loaded. Make a
803 * request that will cause the core to be created if it does not
804 * exist or loaded if it already exists.
805 */
806
807 // In single user mode, if there is a core.properties file already,
808 // we've hit a solr bug. Compensate by deleting it.
810 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
811 if (corePropertiesFile.toFile().exists()) {
812 try {
813 corePropertiesFile.toFile().delete();
814 } catch (Exception ex) {
815 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
816 }
817 }
818 }
819
820 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
821 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
822 createCoreRequest.setCoreName(coreName);
823 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
824 createCoreRequest.setIsLoadOnStartup(false);
825 createCoreRequest.setIsTransient(true);
826 currentSolrServer.request(createCoreRequest);
827 }
828
830 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
831 }
832
833 return new Core(coreName, theCase.getCaseType(), index);
834
835 } catch (Exception ex) {
836 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
837 }
838 }
839
849
850 Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
851 if(serverFilePath.toFile().exists()){
852 try{
853 List<String> lines = Files.readAllLines(serverFilePath);
854 if(lines.isEmpty()) {
855 logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
856 } else if (! lines.get(0).contains(",")) {
857 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
858 } else {
859 String[] parts = lines.get(0).split(",");
860 if(parts.length != 2) {
861 logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
862 } else {
864 }
865 }
866 } catch (IOException ex) {
867 logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
868 }
869 }
870
871 // Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
875 }
876
890 public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
throws KeywordSearchModuleException {
891 // Look for the solr server list file
892 String serverListName = "solrServerList.txt";
893 Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
894 if(serverListPath.toFile().exists()){
895
896 // Read the list of solr servers
897 List<String> lines;
898 try{
899 lines = Files.readAllLines(serverListPath);
900 } catch (IOException ex){
901 throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
902 }
903
904 // Remove any lines that don't contain a comma (these are likely just whitespace)
905 for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
906 String line = iterator.next();
907 if (! line.contains(",")) {
908 // Remove the current element from the iterator and the list.
909 iterator.remove();
910 }
911 }
912 if(lines.isEmpty()) {
913 throw new KeywordSearchModuleException(serverListName + " had no valid server information");
914 }
915
916 // Choose which server to use
917 int rnd = new Random().nextInt(lines.size());
918 String[] parts = lines.get(rnd).split(",");
919 if(parts.length != 2) {
920 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
921 }
922
923 // Split it up just to do a sanity check on the data
924 String host = parts[0];
925 String port = parts[1];
926 if(host.isEmpty() || port.isEmpty()) {
927 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
928 }
929
930 // Write the server data to a file
931 Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
932 try {
933 caseDirectoryPath.toFile().mkdirs();
934 if (! caseDirectoryPath.toFile().exists()) {
935 throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
936 }
937 Files.write(serverFile, lines.get(rnd).getBytes());
938 } catch (IOException ex){
939 throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
940 }
941 }
942 }
943
950
954 }
955
962 }
963
970 }
971 }
972
978 void commit() throws SolrServerException, NoOpenCoreException {
979 currentCoreLock.readLock().lock();
980 try {
981 if (null == currentCore) {
982 throw new NoOpenCoreException();
983 }
984 currentCore.commit();
985 } finally {
986 currentCoreLock.readLock().unlock();
987 }
988 }
989
990 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
991 currentCoreLock.readLock().lock();
992 try {
993 if (null == currentCore) {
994 throw new NoOpenCoreException();
995 }
996 return currentCore.request(request);
997 } finally {
998 currentCoreLock.readLock().unlock();
999 }
1000 }
1001
1013 currentCoreLock.readLock().lock();
1014 try {
1015 if (null == currentCore) {
1016 throw new NoOpenCoreException();
1017 }
1018 try {
1019 return currentCore.queryNumIndexedFiles();
1020 } catch (SolrServerException | IOException ex) {
1021 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
1022 }
1023 } finally {
1024 currentCoreLock.readLock().unlock();
1025 }
1026 }
1027
1038 currentCoreLock.readLock().lock();
1039 try {
1040 if (null == currentCore) {
1041 throw new NoOpenCoreException();
1042 }
1043 try {
1044 return currentCore.queryNumIndexedChunks();
1045 } catch (SolrServerException | IOException ex) {
1046 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
1047 }
1048 } finally {
1049 currentCoreLock.readLock().unlock();
1050 }
1051 }
1052
1063 currentCoreLock.readLock().lock();
1064 try {
1065 if (null == currentCore) {
1066 throw new NoOpenCoreException();
1067 }
1068 try {
1069 return currentCore.queryNumIndexedDocuments();
1070 } catch (SolrServerException | IOException ex) {
1071 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
1072 }
1073 } finally {
1074 currentCoreLock.readLock().unlock();
1075 }
1076 }
1077
1088 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
1089 currentCoreLock.readLock().lock();
1090 try {
1091 if (null == currentCore) {
1092 throw new NoOpenCoreException();
1093 }
1094 try {
1095 return currentCore.queryIsIndexed(contentID);
1096 } catch (SolrServerException | IOException ex) {
1097 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
1098 }
1099
1100 } finally {
1101 currentCoreLock.readLock().unlock();
1102 }
1103 }
1104
1117 currentCoreLock.readLock().lock();
1118 try {
1119 if (null == currentCore) {
1120 throw new NoOpenCoreException();
1121 }
1122 try {
1123 return currentCore.queryNumFileChunks(fileID);
1124 } catch (SolrServerException | IOException ex) {
1125 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
1126 }
1127 } finally {
1128 currentCoreLock.readLock().unlock();
1129 }
1130 }
1131
1142 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1143 currentCoreLock.readLock().lock();
1144 try {
1145 if (null == currentCore) {
1146 throw new NoOpenCoreException();
1147 }
1148 try {
1149 return currentCore.query(sq);
1150 } catch (SolrServerException ex) {
1151 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1152 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1153 }
1154 } finally {
1155 currentCoreLock.readLock().unlock();
1156 }
1157 }
1158
1170 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1171 currentCoreLock.readLock().lock();
1172 try {
1173 if (null == currentCore) {
1174 throw new NoOpenCoreException();
1175 }
1176 try {
1177 return currentCore.query(sq, method);
1178 } catch (SolrServerException | IOException ex) {
1179 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1180 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1181 }
1182 } finally {
1183 currentCoreLock.readLock().unlock();
1184 }
1185 }
1186
1197 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1198 currentCoreLock.readLock().lock();
1199 try {
1200 if (null == currentCore) {
1201 throw new NoOpenCoreException();
1202 }
1203 try {
1204 return currentCore.queryTerms(sq);
1205 } catch (SolrServerException | IOException ex) {
1206 logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1207 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1208 }
1209 } finally {
1210 currentCoreLock.readLock().unlock();
1211 }
1212 }
1213
1224 currentCoreLock.readLock().lock();
1225 try {
1226 if (null == currentCore) {
1227 throw new NoOpenCoreException();
1228 }
1229 return currentCore.getSolrContent(content.getId(), 0);
1230 } finally {
1231 currentCoreLock.readLock().unlock();
1232 }
1233 }
1234
1247 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
1248 currentCoreLock.readLock().lock();
1249 try {
1250 if (null == currentCore) {
1251 throw new NoOpenCoreException();
1252 }
1253 return currentCore.getSolrContent(content.getId(), chunkID);
1254 } finally {
1255 currentCoreLock.readLock().unlock();
1256 }
1257 }
1258
1269 currentCoreLock.readLock().lock();
1270 try {
1271 if (null == currentCore) {
1272 throw new NoOpenCoreException();
1273 }
1274 return currentCore.getSolrContent(objectID, 0);
1275 } finally {
1276 currentCoreLock.readLock().unlock();
1277 }
1278 }
1279
1290 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1291 currentCoreLock.readLock().lock();
1292 try {
1293 if (null == currentCore) {
1294 throw new NoOpenCoreException();
1295 }
1296 return currentCore.getSolrContent(objectID, chunkID);
1297 } finally {
1298 currentCoreLock.readLock().unlock();
1299 }
1300 }
1301
1313 }
1314
1323 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1325 CoreAdminRequest statusRequest = new CoreAdminRequest();
1326 statusRequest.setCoreName( null );
1327 statusRequest.setAction( CoreAdminParams.CoreAdminAction.STATUS );
1328 statusRequest.setIndexInfoNeeded(false);
1329 statusRequest.process(solrServer);
1331 }
1332
1346 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1347 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1348 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1349 }
1350
1362 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1363 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1364 if (null != dataDirPath) {
1365 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1366 return indexDir.exists();
1367 } else {
1368 return false;
1369 }
1370 }
1371
1372 class Core {
1373
1374 // handle to the core in Solr
1375 private final String name;
1376
1378
1379 private final Index textIndex;
1380
1381 // the server to access a core needs to be built from a URL with the
1382 // core in it, and is only good for core-specific operations
1383 private final HttpSolrServer solrCore;
1384
1385 private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
1386
1387 private Core(String name,
CaseType caseType, Index index) {
1388 this.name = name;
1389 this.caseType = caseType;
1390 this.textIndex = index;
1391
1392 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1393
1394 //TODO test these settings
1395 // socket read timeout, make large enough so can index larger files
1396 solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
1397 //solrCore.setConnectionTimeout(1000);
1398 solrCore.setDefaultMaxConnectionsPerHost(32);
1399 solrCore.setMaxTotalConnections(32);
1400 solrCore.setFollowRedirects(false); // defaults to false
1401 // allowCompression defaults to false.
1402 // Server side must support gzip or deflate for this to have any effect.
1403 solrCore.setAllowCompression(true);
1404 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1405
1406 }
1407
1413 String getName() {
1414 return name;
1415 }
1416
1417 private Index getIndexInfo() {
1418 return this.textIndex;
1419 }
1420
1421 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1422 return solrCore.query(sq);
1423 }
1424
1425 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1426 try {
1427 return solrCore.request(request);
1428 } catch (IOException e) {
1429 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1430 throw new SolrServerException(
1431 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1432 }
1433
1434 }
1435
1436 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1437 return solrCore.query(sq, method);
1438 }
1439
1440 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1441 QueryResponse qres = solrCore.query(sq);
1442 return qres.getTermsResponse();
1443 }
1444
1445 private void commit() throws SolrServerException {
1446 try {
1447 //commit and block
1448 solrCore.commit(true, true);
1449 } catch (IOException e) {
1450 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1451 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1452 }
1453 }
1454
1455 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1456 try {
1457 solrCore.add(doc);
1458 } catch (SolrServerException ex) {
1459 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1460 throw new KeywordSearchModuleException(
1461 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1462 } catch (IOException ex) {
1463 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1464 throw new KeywordSearchModuleException(
1465 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1466 }
1467 }
1468
1479 final SolrQuery q = new SolrQuery();
1480 q.setQuery("*:*");
1481 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1482 if (chunkID != 0) {
1483 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1484 }
1485 q.addFilterQuery(filterQuery);
1486 q.setFields(Schema.TEXT.toString());
1487 try {
1488 // Get the first result.
1489 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1490
1491 if (!solrDocuments.isEmpty()) {
1492 SolrDocument solrDocument = solrDocuments.get(0);
1493 if (solrDocument != null) {
1494 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1495 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1496 {
1497 return fieldValues.toArray(new String[0])[0];
1498 } else // The indexed text for files has 2 values, the file name and the file content.
1499 // We return the file content value.
1500 {
1501 return fieldValues.toArray(new String[0])[1];
1502 }
1503 }
1504 }
1505 } catch (SolrServerException ex) {
1506 logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
1507 return null;
1508 }
1509
1510 return null;
1511 }
1512
1513 synchronized void close() throws KeywordSearchModuleException {
1514 // We only unload cores for "single-user" cases.
1516 return;
1517 }
1518
1519 try {
1520 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1521 } catch (SolrServerException ex) {
1522 throw new KeywordSearchModuleException(
1523 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1524 } catch (IOException ex) {
1525 throw new KeywordSearchModuleException(
1526 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1527 }
1528 }
1529
1541 }
1542
1553 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1554 q.setRows(0);
1555 int numChunks = (int)
query(q).getResults().getNumFound();
1556 return numChunks;
1557 }
1558
1570 SolrQuery q = new SolrQuery("*:*");
1571 q.setRows(0);
1572 return (
int)
query(q).getResults().getNumFound();
1573 }
1574
1584 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1585 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1586 SolrQuery q = new SolrQuery("*:*");
1587 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1588 //q.setFields(Server.Schema.ID.toString());
1589 q.setRows(0);
1590 return (
int)
query(q).getResults().getNumFound() != 0;
1591 }
1592
1604 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1605 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1606 final SolrQuery q
1607 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1608 q.setRows(0);
1609 return (
int)
query(q).getResults().getNumFound();
1610 }
1611 }
1612
1613 class ServerAction extends AbstractAction {
1614
1615 private static final long serialVersionUID = 1L;
1616
1617 @Override
1618 public void actionPerformed(ActionEvent e) {
1619 logger.log(Level.INFO, e.paramString().trim());
1620 }
1621 }
1622
1626 class SolrServerNoPortException extends SocketException {
1627
1628 private static final long serialVersionUID = 1L;
1629
1633 private final int port;
1634
1635 SolrServerNoPortException(int port) {
1636 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1637 Server.PROPERTIES_CURRENT_SERVER_PORT));
1638 this.port = port;
1639 }
1640
1641 int getPortNumber() {
1642 return port;
1643 }
1644 }
1645 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
static void submitTimingMetric(TimingMetric metric)
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static IndexingServerProperties getMultiUserServerProperties(String caseDirectory)
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static TimingMetric getTimingMetric(String name)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static final int MAX_SOLR_MEM_MB
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)