1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Path;
37 import java.nio.file.Paths;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.List;
41 import java.util.concurrent.locks.ReentrantReadWriteLock;
42 import java.util.logging.Level;
43 import javax.swing.AbstractAction;
44 import org.apache.solr.client.solrj.SolrQuery;
45 import org.apache.solr.client.solrj.SolrRequest;
46 import org.apache.solr.client.solrj.SolrServerException;
47 import org.apache.solr.client.solrj.impl.HttpSolrServer;
48 import org.apache.solr.client.solrj.impl.XMLResponseParser;
49 import org.apache.solr.client.solrj.request.CoreAdminRequest;
50 import org.apache.solr.client.solrj.response.CoreAdminResponse;
51 import org.apache.solr.client.solrj.response.QueryResponse;
52 import org.apache.solr.client.solrj.response.TermsResponse;
53 import org.apache.solr.common.SolrDocument;
54 import org.apache.solr.common.SolrDocumentList;
55 import org.apache.solr.common.SolrException;
56 import org.apache.solr.common.SolrInputDocument;
57 import org.apache.solr.common.util.NamedList;
58 import org.openide.modules.InstalledFileLocator;
59 import org.openide.modules.Places;
60 import org.openide.util.NbBundle;
69
75
80
81 ID {
82 @Override
83 public String toString() {
84 return "id"; //NON-NLS
85 }
86 },
87 IMAGE_ID {
88 @Override
89 public String toString() {
90 return "image_id"; //NON-NLS
91 }
92 },
93 // This is not stored or index . it is copied to Text and Content_Ws
94 CONTENT {
95 @Override
96 public String toString() {
97 return "content"; //NON-NLS
98 }
99 },
100 TEXT {
101 @Override
102 public String toString() {
103 return "text"; //NON-NLS
104 }
105 },
106 CONTENT_WS {
107 @Override
108 public String toString() {
109 return "content_ws"; //NON-NLS
110 }
111 },
112 FILE_NAME {
113 @Override
114 public String toString() {
115 return "file_name"; //NON-NLS
116 }
117 },
118 // note that we no longer index this field
119 CTIME {
120 @Override
121 public String toString() {
122 return "ctime"; //NON-NLS
123 }
124 },
125 // note that we no longer index this field
126 ATIME {
127 @Override
128 public String toString() {
129 return "atime"; //NON-NLS
130 }
131 },
132 // note that we no longer index this field
133 MTIME {
134 @Override
135 public String toString() {
136 return "mtime"; //NON-NLS
137 }
138 },
139 // note that we no longer index this field
140 CRTIME {
141 @Override
142 public String toString() {
143 return "crtime"; //NON-NLS
144 }
145 },
146 NUM_CHUNKS {
147 @Override
148 public String toString() {
149 return "num_chunks"; //NON-NLS
150 }
151 },
152 };
153
154 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
155 //max content size we can send to Solr
159 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
160 @Deprecated
165 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
167 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
168 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
169 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
170 private static final String
KEY =
"jjk#09s";
//NON-NLS
171 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
172 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
173 static final int DEFAULT_SOLR_STOP_PORT = 34343;
176 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
178 private static final String
SOLR =
"solr";
180
182
185
186 // A reference to the locally running Solr instance.
188
189 // A reference to the Solr server we are currently connected to for the Case.
190 // This could be a local or remote server.
192
195
199
206
207 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
208 serverAction = new ServerAction();
209 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
211
212 currentCoreLock = new ReentrantReadWriteLock(true);
214
215 logger.log(Level.INFO, "Created Server instance"); //NON-NLS
216 }
217
219
221 try {
223 } catch (NumberFormatException nfe) {
224 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
225 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
226 }
227 } else {
228 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
230 }
231
233 try {
235 } catch (NumberFormatException nfe) {
236 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
237 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
238 }
239 } else {
240 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
242 }
243 }
244
245 @Override
246 public void finalize() throws java.lang.Throwable {
247 stop();
248 super.finalize();
249 }
250
252 serverAction.addPropertyChangeListener(l);
253 }
254
255 int getCurrentSolrServerPort() {
257 }
258
259 int getCurrentSolrStopPort() {
261 }
262
267
268 InputStream stream;
269 OutputStream out;
270 volatile boolean doRun = true;
271
273 this.stream = stream;
274 try {
275 final String log = Places.getUserDirectory().getAbsolutePath()
276 + File.separator + "var" + File.separator + "log" //NON-NLS
277 + File.separator + "solr.log." + type; //NON-NLS
278 File outputFile = new File(log.concat(".0"));
279 File first = new File(log.concat(".1"));
280 File second = new File(log.concat(".2"));
281 if (second.exists()) {
282 second.delete();
283 }
284 if (first.exists()) {
285 first.renameTo(second);
286 }
287 if (outputFile.exists()) {
288 outputFile.renameTo(first);
289 } else {
290 outputFile.createNewFile();
291 }
292 out = new FileOutputStream(outputFile);
293
294 } catch (Exception ex) {
295 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
296 }
297 }
298
299 void stopRun() {
300 doRun = false;
301 }
302
303 @Override
305
306 try (InputStreamReader isr = new InputStreamReader(stream);
307 BufferedReader br = new BufferedReader(isr);
309 BufferedWriter bw = new BufferedWriter(osw);) {
310
311 String line = null;
312 while (doRun && (line = br.readLine()) != null) {
313 bw.write(line);
314 bw.newLine();
315 if (DEBUG) {
316 //flush buffers if dev version for debugging
317 bw.flush();
318 }
319 }
320 bw.flush();
321 } catch (IOException ex) {
322 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
323 }
324 }
325 }
326
332 List<Long> getSolrPIDs() {
333 List<Long> pids = new ArrayList<>();
334
335 //NOTE: these needs to be in sync with process start string in start()
336 final String pidsQuery = "Args.4.eq=-DSTOP.KEY=" + KEY + ",Args.6.eq=start.jar"; //NON-NLS
337
339 if (pidsArr != null) {
340 for (int i = 0; i < pidsArr.length; ++i) {
341 pids.add(pidsArr[i]);
342 }
343 }
344
345 return pids;
346 }
347
352 void killSolr() {
353 List<Long> solrPids = getSolrPIDs();
354 for (long pid : solrPids) {
355 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
356 PlatformUtil.killProcess(pid);
357 }
358 }
359
365 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
366 if (isRunning()) {
367 // If a Solr server is running we stop it.
368 stop();
369 }
370
371 if (!isPortAvailable(currentSolrServerPort)) {
372 // There is something already listening on our port. Let's see if
373 // this is from an earlier run that didn't successfully shut down
374 // and if so kill it.
375 final List<Long> pids = this.getSolrPIDs();
376
377 // If the culprit listening on the port is not a Solr process
378 // we refuse to start.
379 if (pids.isEmpty()) {
380 throw new SolrServerNoPortException(currentSolrServerPort);
381 }
382
383 // Ok, we've tried to stop it above but there still appears to be
384 // a Solr process listening on our port so we forcefully kill it.
385 killSolr();
386
387 // If either of the ports are still in use after our attempt to kill
388 // previously running processes we give up and throw an exception.
389 if (!isPortAvailable(currentSolrServerPort)) {
390 throw new SolrServerNoPortException(currentSolrServerPort);
391 }
392 if (!isPortAvailable(currentSolrStopPort)) {
393 throw new SolrServerNoPortException(currentSolrStopPort);
394 }
395 }
396
397 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
398
399 if (isPortAvailable(currentSolrServerPort)) {
400 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
401 try {
402 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
403 List<String> commandLine = new ArrayList<>();
404 commandLine.add(javaPath);
405 commandLine.add(MAX_SOLR_MEM_MB_PAR);
406 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
407 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
408 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
409 commandLine.add("-jar"); //NON-NLS
410 commandLine.add("start.jar"); //NON-NLS
411
412 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
413 solrProcessBuilder.directory(solrFolder);
414
415 // Redirect stdout and stderr to files to prevent blocking.
416 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
417 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
418
419 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
420 solrProcessBuilder.redirectError(solrStderrPath.toFile());
421
422 logger.log(Level.INFO, "Starting Solr using: {0}", solrProcessBuilder.command()); //NON-NLS
423 curSolrProcess = solrProcessBuilder.start();
424 logger.log(Level.INFO, "Finished starting Solr"); //NON-NLS
425
426 try {
427 //block for 10 seconds, give time to fully start the process
428 //so if it's restarted solr operations can be resumed seamlessly
429 Thread.sleep(10 * 1000);
430 } catch (InterruptedException ex) {
431 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
432 }
433
434 final List<Long> pids = this.getSolrPIDs();
435 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
436 } catch (SecurityException ex) {
437 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
438 throw new KeywordSearchModuleException(
439 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
440 } catch (IOException ex) {
441 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
442 throw new KeywordSearchModuleException(
443 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
444 }
445 }
446 }
447
453 static boolean isPortAvailable(int port) {
454 ServerSocket ss = null;
455 try {
456
457 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
458 if (ss.isBound()) {
459 ss.setReuseAddress(true);
460 ss.close();
461 return true;
462 }
463
464 } catch (IOException e) {
465 } finally {
466 if (ss != null) {
467 try {
468 ss.close();
469 } catch (IOException e) {
470 /*
471 * should not be thrown
472 */
473 }
474 }
475 }
476 return false;
477 }
478
484 void changeSolrServerPort(int port) {
485 currentSolrServerPort = port;
486 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
487 }
488
494 void changeSolrStopPort(int port) {
495 currentSolrStopPort = port;
496 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
497 }
498
504 synchronized void stop() {
505
506 try {
507 // Close any open core before stopping server
508 closeCore();
509 } catch (KeywordSearchModuleException e) {
510 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
511 }
512
513 try {
514 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
515
516 //try graceful shutdown
517 final String[] SOLR_STOP_CMD = {
520 "-DSTOP.KEY=" +
KEY,
//NON-NLS
521 "-jar", //NON-NLS
522 "start.jar", //NON-NLS
523 "--stop", //NON-NLS
524 };
525 Process stop = Runtime.getRuntime().exec(SOLR_STOP_CMD, null, solrFolder);
526 logger.log(Level.INFO, "Waiting for stopping Solr server"); //NON-NLS
527 stop.waitFor();
528
529 //if still running, forcefully stop it
530 if (curSolrProcess != null) {
531 curSolrProcess.destroy();
532 curSolrProcess = null;
533 }
534
535 } catch (InterruptedException | IOException ex) {
536 } finally {
537 //stop Solr stream -> log redirect threads
538 try {
539 if (errorRedirectThread != null) {
540 errorRedirectThread.stopRun();
541 errorRedirectThread = null;
542 }
543 } finally {
544 //if still running, kill it
545 killSolr();
546 }
547
548 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
549 }
550 }
551
559 synchronized boolean isRunning() throws KeywordSearchModuleException {
560 try {
561
562 if (isPortAvailable(currentSolrServerPort)) {
563 return false;
564 }
565
566 if (curSolrProcess != null && !curSolrProcess.isAlive()) {
567 return false;
568 }
569
570 // making a status request here instead of just doing solrServer.ping(), because
571 // that doesn't work when there are no cores
572 //TODO handle timeout in cases when some other type of server on that port
573 CoreAdminRequest.getStatus(null, localSolrServer);
574
575 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
576 } catch (SolrServerException ex) {
577
578 Throwable cause = ex.getRootCause();
579
580 // TODO: check if SocketExceptions should actually happen (is
581 // probably caused by starting a connection as the server finishes
582 // shutting down)
583 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
584 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
585 return false;
586 } else {
587 throw new KeywordSearchModuleException(
588 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
589 }
590 } catch (SolrException ex) {
591 // Just log 404 errors for now...
592 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
593 return false;
594 } catch (IOException ex) {
595 throw new KeywordSearchModuleException(
596 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
597 }
598
599 return true;
600 }
601
602 /*
603 * ** Convenience methods for use while we only open one case at a time ***
604 */
614 void openCoreForCase(Case theCase) throws KeywordSearchModuleException {
615 currentCoreLock.writeLock().lock();
616 try {
618 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
619 } finally {
620 currentCoreLock.writeLock().unlock();
621 }
622 }
623
629 boolean coreIsOpen() {
630 currentCoreLock.readLock().lock();
631 try {
632 return (null != currentCore);
633 } finally {
634 currentCoreLock.readLock().unlock();
635 }
636 }
637
638 void closeCore() throws KeywordSearchModuleException {
639 currentCoreLock.writeLock().lock();
640 try {
641 if (null != currentCore) {
642 currentCore.close();
643 currentCore = null;
644 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
645 }
646 } finally {
647 currentCoreLock.writeLock().unlock();
648 }
649 }
650
651 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
652 currentCoreLock.readLock().lock();
653 try {
654 currentCore.addDocument(doc);
655 } finally {
656 currentCoreLock.readLock().unlock();
657 }
658 }
659
667 String geCoreDataDirPath(Case theCase) {
668 String indexDir = theCase.getModuleDirectory() + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS
669 if (uncPathUtilities != null) {
670 // if we can check for UNC paths, do so, otherwise just return the indexDir
672 if (result == null) {
675 }
676 if (result == null) {
677 return indexDir;
678 }
679 return result;
680 }
681 return indexDir;
682 }
683
697 private Core
openCore(
Case theCase)
throws KeywordSearchModuleException {
698 try {
701 } else {
704 currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
705 }
706 connectToSolrServer(currentSolrServer);
707
708 } catch (SolrServerException | IOException ex) {
709 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg"), ex);
710 }
711
712 String dataDir = geCoreDataDirPath(theCase);
713 String coreName = theCase.getTextIndexName();
714 return this.
openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName,
new File(dataDir), theCase.getCaseType());
715 }
716
723 currentCoreLock.readLock().lock();
724 try {
725 if (null == currentCore) {
726 throw new NoOpenCoreException();
727 }
728 currentCore.commit();
729 } finally {
730 currentCoreLock.readLock().unlock();
731 }
732 }
733
734 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
735 currentCoreLock.readLock().lock();
736 try {
737 if (null == currentCore) {
738 throw new NoOpenCoreException();
739 }
740 return currentCore.request(request);
741 } finally {
742 currentCoreLock.readLock().unlock();
743 }
744 }
745
757 currentCoreLock.readLock().lock();
758 try {
759 if (null == currentCore) {
760 throw new NoOpenCoreException();
761 }
762 try {
763 return currentCore.queryNumIndexedFiles();
764 } catch (SolrServerException ex) {
765 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
766 }
767 } finally {
768 currentCoreLock.readLock().unlock();
769 }
770 }
771
782 currentCoreLock.readLock().lock();
783 try {
784 if (null == currentCore) {
785 throw new NoOpenCoreException();
786 }
787 try {
788 return currentCore.queryNumIndexedChunks();
789 } catch (SolrServerException ex) {
790 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
791 }
792 } finally {
793 currentCoreLock.readLock().unlock();
794 }
795 }
796
807 currentCoreLock.readLock().lock();
808 try {
809 if (null == currentCore) {
810 throw new NoOpenCoreException();
811 }
812 try {
813 return currentCore.queryNumIndexedDocuments();
814 } catch (SolrServerException ex) {
815 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
816 }
817 } finally {
818 currentCoreLock.readLock().unlock();
819 }
820 }
821
832 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
833 currentCoreLock.readLock().lock();
834 try {
835 if (null == currentCore) {
836 throw new NoOpenCoreException();
837 }
838 try {
839 return currentCore.queryIsIndexed(contentID);
840 } catch (SolrServerException ex) {
841 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
842 }
843
844 } finally {
845 currentCoreLock.readLock().unlock();
846 }
847 }
848
860 public int queryNumFileChunks(
long fileID)
throws KeywordSearchModuleException, NoOpenCoreException {
861 currentCoreLock.readLock().lock();
862 try {
863 if (null == currentCore) {
864 throw new NoOpenCoreException();
865 }
866 try {
867 return currentCore.queryNumFileChunks(fileID);
868 } catch (SolrServerException ex) {
869 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
870 }
871 } finally {
872 currentCoreLock.readLock().unlock();
873 }
874 }
875
886 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
887 currentCoreLock.readLock().lock();
888 try {
889 if (null == currentCore) {
890 throw new NoOpenCoreException();
891 }
892 try {
893 return currentCore.query(sq);
894 } catch (SolrServerException ex) {
895 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
896 }
897 } finally {
898 currentCoreLock.readLock().unlock();
899 }
900 }
901
913 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
914 currentCoreLock.readLock().lock();
915 try {
916 if (null == currentCore) {
917 throw new NoOpenCoreException();
918 }
919 try {
920 return currentCore.query(sq, method);
921 } catch (SolrServerException ex) {
922 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
923 }
924 } finally {
925 currentCoreLock.readLock().unlock();
926 }
927 }
928
939 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
940 currentCoreLock.readLock().lock();
941 try {
942 if (null == currentCore) {
943 throw new NoOpenCoreException();
944 }
945 try {
946 return currentCore.queryTerms(sq);
947 } catch (SolrServerException ex) {
948 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
949 }
950 } finally {
951 currentCoreLock.readLock().unlock();
952 }
953 }
954
965 currentCoreLock.readLock().lock();
966 try {
967 if (null == currentCore) {
968 throw new NoOpenCoreException();
969 }
970 return currentCore.getSolrContent(content.getId(), 0);
971 } finally {
972 currentCoreLock.readLock().unlock();
973 }
974 }
975
988 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
989 currentCoreLock.readLock().lock();
990 try {
991 if (null == currentCore) {
992 throw new NoOpenCoreException();
993 }
994 return currentCore.getSolrContent(content.getId(), chunkID);
995 } finally {
996 currentCoreLock.readLock().unlock();
997 }
998 }
999
1010 currentCoreLock.readLock().lock();
1011 try {
1012 if (null == currentCore) {
1013 throw new NoOpenCoreException();
1014 }
1015 return currentCore.getSolrContent(objectID, 0);
1016 } finally {
1017 currentCoreLock.readLock().unlock();
1018 }
1019 }
1020
1031 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1032 currentCoreLock.readLock().lock();
1033 try {
1034 if (null == currentCore) {
1035 throw new NoOpenCoreException();
1036 }
1037 return currentCore.getSolrContent(objectID, chunkID);
1038 } finally {
1039 currentCoreLock.readLock().unlock();
1040 }
1041 }
1042
1049 return Ingester.getDefault();
1050 }
1051
1063 }
1064
1078 private Core
openCore(String coreName, File dataDir,
CaseType caseType)
throws KeywordSearchModuleException {
1079
1080 try {
1081 if (!dataDir.exists()) {
1082 dataDir.mkdirs();
1083 }
1084
1085 if (!this.isRunning()) {
1086 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
1087 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
1088 }
1089
1091 /*
1092 * The core either does not exist or it is not loaded. Make a
1093 * request that will cause the core to be created if it does not
1094 * exist or loaded if it already exists.
1095 */
1096
1097 // In single user mode, if there is a core.properties file already,
1098 // we've hit a solr bug. Compensate by deleting it.
1100 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
1101 if (corePropertiesFile.toFile().exists()) {
1102 try {
1103 corePropertiesFile.toFile().delete();
1104 } catch (Exception ex) {
1105 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
1106 }
1107 }
1108 }
1109
1110 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
1111 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
1112 createCoreRequest.setCoreName(coreName);
1113 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
1114 createCoreRequest.setIsLoadOnStartup(false);
1115 createCoreRequest.setIsTransient(true);
1116 currentSolrServer.request(createCoreRequest);
1117 }
1118
1120 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1121 }
1122
1123 return new Core(coreName, caseType);
1124
1125 } catch (SolrServerException | SolrException | IOException ex) {
1126 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1127 }
1128 }
1129
1138 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1139 CoreAdminRequest.getStatus(null, solrServer);
1140 }
1141
1155 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1156 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1157 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1158 }
1159
1171 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1172 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1173 if (null != dataDirPath) {
1174 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1175 return indexDir.exists();
1176 } else {
1177 return false;
1178 }
1179 }
1180
1181 class Core {
1182
1183 // handle to the core in Solr
1184 private final String name;
1185
1187
1188 // the server to access a core needs to be built from a URL with the
1189 // core in it, and is only good for core-specific operations
1190 private final HttpSolrServer solrCore;
1191
1192 private Core(String name,
CaseType caseType) {
1193 this.name = name;
1194 this.caseType = caseType;
1195
1196 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name);
1197
1198 //TODO test these settings
1199 //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
1200 //solrCore.setConnectionTimeout(1000);
1201 solrCore.setDefaultMaxConnectionsPerHost(2);
1202 solrCore.setMaxTotalConnections(5);
1203 solrCore.setFollowRedirects(false); // defaults to false
1204 // allowCompression defaults to false.
1205 // Server side must support gzip or deflate for this to have any effect.
1206 solrCore.setAllowCompression(true);
1207 solrCore.setMaxRetries(1); // defaults to 0. > 1 not recommended.
1208 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1209
1210 }
1211
1217 String getName() {
1218 return name;
1219 }
1220
1221 private QueryResponse
query(SolrQuery sq)
throws SolrServerException {
1222 return solrCore.query(sq);
1223 }
1224
1225 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1226 try {
1227 return solrCore.request(request);
1228 } catch (IOException e) {
1229 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1230 throw new SolrServerException(
1231 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1232 }
1233
1234 }
1235
1236 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException {
1237 return solrCore.query(sq, method);
1238 }
1239
1240 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException {
1241 QueryResponse qres = solrCore.query(sq);
1242 return qres.getTermsResponse();
1243 }
1244
1245 private void commit() throws SolrServerException {
1246 try {
1247 //commit and block
1248 solrCore.commit(true, true);
1249 } catch (IOException e) {
1250 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1251 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1252 }
1253 }
1254
1255 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1256 try {
1257 solrCore.add(doc);
1258 } catch (SolrServerException ex) {
1259 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1260 throw new KeywordSearchModuleException(
1261 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1262 } catch (IOException ex) {
1263 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1264 throw new KeywordSearchModuleException(
1265 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1266 }
1267 }
1268
1278 final SolrQuery q = new SolrQuery();
1279 q.setQuery("*:*");
1280 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1281 if (chunkID != 0) {
1282 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1283 }
1284 q.addFilterQuery(filterQuery);
1285 q.setFields(Schema.TEXT.toString());
1286 try {
1287 // Get the first result.
1288 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1289
1290 if (!solrDocuments.isEmpty()) {
1291 SolrDocument solrDocument = solrDocuments.get(0);
1292 if (solrDocument != null) {
1293 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1294 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1295 {
1296 return fieldValues.toArray(new String[0])[0];
1297 } else // The indexed text for files has 2 values, the file name and the file content.
1298 // We return the file content value.
1299 {
1300 return fieldValues.toArray(new String[0])[1];
1301 }
1302 }
1303 }
1304 } catch (SolrServerException ex) {
1305 logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1306 return null;
1307 }
1308
1309 return null;
1310 }
1311
1312 synchronized void close() throws KeywordSearchModuleException {
1313 // We only unload cores for "single-user" cases.
1315 return;
1316 }
1317
1318 try {
1319 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1320 } catch (SolrServerException ex) {
1321 throw new KeywordSearchModuleException(
1322 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1323 } catch (IOException ex) {
1324 throw new KeywordSearchModuleException(
1325 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1326 }
1327 }
1328
1340 }
1341
1351 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1352 q.setRows(0);
1353 int numChunks = (int)
query(q).getResults().getNumFound();
1354 return numChunks;
1355 }
1356
1368 SolrQuery q = new SolrQuery("*:*");
1369 q.setRows(0);
1370 return (
int)
query(q).getResults().getNumFound();
1371 }
1372
1382 private boolean queryIsIndexed(
long contentID)
throws SolrServerException {
1383 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1384 SolrQuery q = new SolrQuery("*:*");
1385 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1386 //q.setFields(Server.Schema.ID.toString());
1387 q.setRows(0);
1388 return (
int)
query(q).getResults().getNumFound() != 0;
1389 }
1390
1403 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1404 final SolrQuery q
1405 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1406 q.setRows(0);
1407 return (
int)
query(q).getResults().getNumFound();
1408 }
1409 }
1410
1411 class ServerAction extends AbstractAction {
1412
1413 private static final long serialVersionUID = 1L;
1414
1415 @Override
1416 public void actionPerformed(ActionEvent e) {
1417 logger.log(Level.INFO, e.paramString().trim());
1418 }
1419 }
1420
1424 class SolrServerNoPortException extends SocketException {
1425
1426 private static final long serialVersionUID = 1L;
1427
1431 private final int port;
1432
1433 SolrServerNoPortException(int port) {
1434 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1435 Server.PROPERTIES_CURRENT_SERVER_PORT));
1436 this.port = port;
1437 }
1438
1439 int getPortNumber() {
1440 return port;
1441 }
1442 }
1443 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
synchronized void rescanDrives()
final ReentrantReadWriteLock currentCoreLock
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
synchronized String mappedDriveToUNC(String inputPath)
UNCPathUtilities uncPathUtilities
static String getIndexingServerPort()
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
Core openCore(Case theCase)
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
static final String DEFAULT_CORE_NAME
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(String coreName, File dataDir, CaseType caseType)
static final int MAX_SOLR_MEM_MB
static Ingester getIngester()
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)