1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2015 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Paths;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.List;
41 import java.util.logging.Level;
42
43 import org.openide.util.NbBundle;
45 import javax.swing.AbstractAction;
46 import org.apache.solr.client.solrj.SolrQuery;
47 import org.apache.solr.client.solrj.SolrServer;
48 import org.apache.solr.client.solrj.SolrServerException;
49 import org.apache.solr.client.solrj.request.CoreAdminRequest;
50 import org.apache.solr.client.solrj.response.QueryResponse;
51 import org.apache.solr.client.solrj.response.TermsResponse;
52 import org.apache.solr.client.solrj.SolrRequest;
53 import org.apache.solr.client.solrj.impl.HttpSolrServer;
54 import org.apache.solr.common.util.NamedList;
55 import org.openide.modules.InstalledFileLocator;
56 import org.openide.modules.Places;
61 import org.apache.solr.common.SolrInputDocument;
62 import org.apache.solr.client.solrj.impl.XMLResponseParser;
63 import org.apache.solr.common.SolrDocument;
64 import org.apache.solr.common.SolrException;
65
70
71 // field names that are used in SOLR schema
73 ID {
74 @Override
75 public String toString() {
76 return "id"; //NON-NLS
77 }
78 },
79 IMAGE_ID {
80 @Override
81 public String toString() {
82 return "image_id"; //NON-NLS
83 }
84 },
85 // This is not stored or index . it is copied to Text and Content_Ws
86 CONTENT {
87 @Override
88 public String toString() {
89 return "content"; //NON-NLS
90 }
91 },
92 TEXT {
93 @Override
94 public String toString() {
95 return "text"; //NON-NLS
96 }
97 },
98 CONTENT_WS {
99 @Override
100 public String toString() {
101 return "content_ws"; //NON-NLS
102 }
103 },
104 FILE_NAME {
105 @Override
106 public String toString() {
107 return "file_name"; //NON-NLS
108 }
109 },
110 // note that we no longer index this field
111 CTIME {
112 @Override
113 public String toString() {
114 return "ctime"; //NON-NLS
115 }
116 },
117 // note that we no longer index this field
118 ATIME {
119 @Override
120 public String toString() {
121 return "atime"; //NON-NLS
122 }
123 },
124 // note that we no longer index this field
125 MTIME {
126 @Override
127 public String toString() {
128 return "mtime"; //NON-NLS
129 }
130 },
131 // note that we no longer index this field
132 CRTIME {
133 @Override
134 public String toString() {
135 return "crtime"; //NON-NLS
136 }
137 },
138 NUM_CHUNKS {
139 @Override
140 public String toString() {
141 return "num_chunks"; //NON-NLS
142 }
143 },
144 };
145 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
146 //max content size we can send to Solr
150 // TODO: DEFAULT_CORE_NAME needs to be replaced with unique names to support multiple open cases
151 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
155 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
158 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
159 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
160 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
161 private static final String
KEY =
"jjk#09s";
//NON-NLS
162 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
163 static final int DEFAULT_SOLR_STOP_PORT = 34343;
166 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
167
169
178
186
187 this.solrUrl = "http://localhost:" + currentSolrServerPort + "/solr"; //NON-NLS
188 this.solrServer = new HttpSolrServer(solrUrl);
189 serverAction = new ServerAction();
190 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
191 instanceDir = solrFolder.getAbsolutePath() +
File.separator +
"solr";
//NON-NLS
193
194 logger.log(Level.INFO, "Created Server instance"); //NON-NLS
195 }
196
199 try {
201 } catch (NumberFormatException nfe) {
202 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
203 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
204 }
205 } else {
206 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
208 }
209
211 try {
213 } catch (NumberFormatException nfe) {
214 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
215 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
216 }
217 } else {
218 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
220 }
221 }
222
223 @Override
224 public void finalize() throws java.lang.Throwable {
225 stop();
226 super.finalize();
227 }
228
230 serverAction.addPropertyChangeListener(l);
231 }
232
233 int getCurrentSolrServerPort() {
235 }
236
237 int getCurrentSolrStopPort() {
239 }
240
245
246 InputStream stream;
247 OutputStream out;
248 volatile boolean doRun = true;
249
251 this.stream = stream;
252 try {
253 final String log = Places.getUserDirectory().getAbsolutePath()
254 +
File.separator +
"var" +
File.separator +
"log" //NON-NLS
255 +
File.separator +
"solr.log." + type;
//NON-NLS
256 File outputFile =
new File(log.concat(
".0"));
257 File first =
new File(log.concat(
".1"));
258 File second =
new File(log.concat(
".2"));
259 if (second.exists()) {
260 second.delete();
261 }
262 if (first.exists()) {
263 first.renameTo(second);
264 }
265 if (outputFile.
exists()) {
266 outputFile.renameTo(first);
267 } else {
268 outputFile.createNewFile();
269 }
270 out = new FileOutputStream(outputFile);
271
272 } catch (Exception ex) {
273 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
274 }
275 }
276
277 void stopRun() {
278 doRun = false;
279 }
280
281 @Override
283 InputStreamReader isr = new InputStreamReader(stream);
284 BufferedReader br = new BufferedReader(isr);
285 OutputStreamWriter osw = null;
286 BufferedWriter bw = null;
287 try {
289 bw = new BufferedWriter(osw);
290 String line = null;
291 while (doRun && (line = br.readLine()) != null) {
292 bw.write(line);
293 bw.newLine();
294 if (DEBUG) {
295 //flush buffers if dev version for debugging
296 bw.flush();
297 }
298 }
299 bw.flush();
300 } catch (IOException ex) {
301 logger.log(Level.WARNING, "Error redirecting Solr output stream"); //NON-NLS
302 } finally {
303 if (bw != null) {
304 try {
305 bw.close();
306 } catch (IOException ex) {
307 logger.log(Level.WARNING, "Error closing Solr output stream writer"); //NON-NLS
308 }
309 }
310 if (br != null) {
311 try {
312 br.close();
313 } catch (IOException ex) {
314 logger.log(Level.WARNING, "Error closing Solr output stream reader"); //NON-NLS
315 }
316 }
317 }
318 }
319 }
320
326 List<Long> getSolrPIDs() {
327 List<Long> pids = new ArrayList<Long>();
328
329 //NOTE: these needs to be in sync with process start string in start()
330 final String pidsQuery = "Args.4.eq=-DSTOP.KEY=" + KEY + ",Args.7.eq=start.jar"; //NON-NLS
331
333 if (pidsArr != null) {
334 for (int i = 0; i < pidsArr.length; ++i) {
335 pids.add(pidsArr[i]);
336 }
337 }
338
339 return pids;
340 }
341
346 void killSolr() {
347 List<Long> solrPids = getSolrPIDs();
348 for (long pid : solrPids) {
349 logger.log(Level.INFO, "Trying to kill old Solr process, PID: " + pid); //NON-NLS
350 PlatformUtil.killProcess(pid);
351 }
352 }
353
359 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
360 logger.log(Level.INFO, "Starting Solr server from: " + solrFolder.getAbsolutePath()); //NON-NLS
361 if (isPortAvailable(currentSolrServerPort)) {
362 logger.log(Level.INFO, "Port [" + currentSolrServerPort + "] available, starting Solr"); //NON-NLS
363 try {
364 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
365
366 String loggingPropertiesOpt = "-Djava.util.logging.config.file="; //NON-NLS
367 String loggingPropertiesFilePath = instanceDir + File.separator + "conf" + File.separator; //NON-NLS
368
369 if (DEBUG) {
370 loggingPropertiesFilePath += "logging-development.properties"; //NON-NLS
371 } else {
372 loggingPropertiesFilePath += "logging-release.properties"; //NON-NLS
373 }
374
375 final String loggingProperties = loggingPropertiesOpt + loggingPropertiesFilePath;
376
377 final String [] SOLR_START_CMD = {
379 MAX_SOLR_MEM_MB_PAR,
382 "-DSTOP.KEY=" +
KEY,
//NON-NLS
383 loggingProperties,
384 "-jar", //NON-NLS
385 "start.jar"}; //NON-NLS
386
387 StringBuilder cmdSb = new StringBuilder();
388 for (int i = 0; i<SOLR_START_CMD.length; ++i ) {
389 cmdSb.append(SOLR_START_CMD[i]).append(" ");
390 }
391
392 logger.log(Level.INFO, "Starting Solr using: " + cmdSb.toString()); //NON-NLS
393 curSolrProcess = Runtime.getRuntime().exec(SOLR_START_CMD, null, solrFolder);
394 logger.log(Level.INFO, "Finished starting Solr"); //NON-NLS
395
396 try {
397 //block for 10 seconds, give time to fully start the process
398 //so if it's restarted solr operations can be resumed seamlessly
399 Thread.sleep(10 * 1000);
400 } catch (InterruptedException ex) {
401 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
402 }
403 // Handle output to prevent process from blocking
404
405 errorRedirectThread = new InputStreamPrinterThread(curSolrProcess.getErrorStream(), "stderr"); //NON-NLS
406 errorRedirectThread.start();
407
408 final List<Long> pids = this.getSolrPIDs();
409 logger.log(Level.INFO, "New Solr process PID: " + pids); //NON-NLS
410 } catch (SecurityException ex) {
411 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
412 throw new KeywordSearchModuleException(
413 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
414 } catch (IOException ex) {
415 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
416 throw new KeywordSearchModuleException(
417 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
418 }
419 } else {
420 logger.log(Level.SEVERE, "Could not start Solr server process, port [" + currentSolrServerPort + "] not available!"); //NON-NLS
421 throw new SolrServerNoPortException(currentSolrServerPort);
422 }
423 }
424
430 static boolean isPortAvailable(int port) {
431 ServerSocket ss = null;
432 try {
433
434 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
435 if (ss.isBound()) {
436 ss.setReuseAddress(true);
437 ss.close();
438 return true;
439 }
440
441 } catch (IOException e) {
442 } finally {
443 if (ss != null) {
444 try {
445 ss.close();
446 } catch (IOException e) {
447 /* should not be thrown */
448 }
449 }
450 }
451 return false;
452 }
453
459 void changeSolrServerPort(int port) {
460 currentSolrServerPort = port;
461 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
462 }
463
469 void changeSolrStopPort(int port) {
470 currentSolrStopPort = port;
471 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
472 }
473
479 synchronized void stop() {
480 try {
481 logger.log(Level.INFO, "Stopping Solr server from: " + solrFolder.getAbsolutePath()); //NON-NLS
482 //try graceful shutdown
483 final String [] SOLR_STOP_CMD = {
486 "-DSTOP.KEY=" +
KEY,
//NON-NLS
487 "-jar", //NON-NLS
488 "start.jar", //NON-NLS
489 "--stop", //NON-NLS
490 };
491 Process stop = Runtime.getRuntime().exec(SOLR_STOP_CMD, null, solrFolder);
492 logger.log(Level.INFO, "Waiting for stopping Solr server"); //NON-NLS
493 stop.waitFor();
494
495 //if still running, forcefully stop it
496 if (curSolrProcess != null) {
497 curSolrProcess.destroy();
498 curSolrProcess = null;
499 }
500
501 } catch (InterruptedException ex) {
502 } catch (IOException ex) {
503 } finally {
504 //stop Solr stream -> log redirect threads
505 try {
506 if (errorRedirectThread != null) {
507 errorRedirectThread.stopRun();
508 errorRedirectThread = null;
509 }
510 } finally {
511 //if still running, kill it
512 killSolr();
513 }
514
515 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
516 }
517 }
518
526 synchronized boolean isRunning() throws KeywordSearchModuleException {
527 try {
528 // making a status request here instead of just doing solrServer.ping(), because
529 // that doesn't work when there are no cores
530
531 //TODO check if port avail and return false if it is
532
533 //TODO handle timeout in cases when some other type of server on that port
534 CoreAdminRequest.getStatus(null, solrServer);
535
536 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
537 } catch (SolrServerException ex) {
538
539 Throwable cause = ex.getRootCause();
540
541 // TODO: check if SocketExceptions should actually happen (is
542 // probably caused by starting a connection as the server finishes
543 // shutting down)
544 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
545 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
546 return false;
547 } else {
548 throw new KeywordSearchModuleException(
549 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
550 }
551 } catch (SolrException ex) {
552 // Just log 404 errors for now...
553 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
554 return false;
555 } catch (IOException ex) {
556 throw new KeywordSearchModuleException(
557 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
558 }
559
560 return true;
561 }
566
567 synchronized void openCore() throws KeywordSearchModuleException {
568 if (currentCore != null) {
569 throw new KeywordSearchModuleException(
570 NbBundle.getMessage(this.getClass(), "Server.openCore.exception.alreadyOpen.msg"));
571 }
572
574
576
577 currentCore = openCore(currentCase);
579 }
580
586 logger.log(Level.INFO, "Validating keyword search index location"); //NON-NLS
587 String properIndexPath = getIndexDirPath(theCase);
588
590 +
File.separator +
"keywordsearch" +
File.separator +
"data";
//NON-NLS
591
592
593 File properIndexDir =
new File(properIndexPath);
594 File legacyIndexDir =
new File(legacyIndexPath);
595 if (!properIndexDir.
exists()
596 && legacyIndexDir.
exists() && legacyIndexDir.isDirectory()) {
597 logger.log(Level.INFO, "Moving keyword search index location from: " //NON-NLS
598 + legacyIndexPath + " to: " + properIndexPath); //NON-NLS
599 try {
600 Files.move(Paths.get(legacyIndexDir.
getParent()), Paths.get(properIndexDir.
getParent()));
601 } catch (IOException | SecurityException ex) {
602 logger.log(Level.WARNING, "Error moving keyword search index folder from: " //NON-NLS
603 + legacyIndexPath + " to: " + properIndexPath //NON-NLS
604 + " will recreate a new index.", ex); //NON-NLS
605 }
606 }
607 }
608
609 synchronized void closeCore() throws KeywordSearchModuleException {
610 if (currentCore == null) {
611 return;
612 }
613 currentCore.close();
614 currentCore = null;
615 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
616 }
617
618 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
619 currentCore.addDocument(doc);
620 }
621
628 String getIndexDirPath(Case theCase) {
629 String indexDir = theCase.getModulesOutputDirAbsPath()
630 + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS
631 return indexDir;
632 }
633
643 private synchronized Core
openCore(
Case theCase)
throws KeywordSearchModuleException {
644 String dataDir = getIndexDirPath(theCase);
645 return this.openCore(DEFAULT_CORE_NAME,
new File(dataDir));
646 }
647
654 if (currentCore == null) {
655 throw new NoOpenCoreException();
656 }
657 currentCore.commit();
658 }
659
660 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
661 if (currentCore == null) {
662 throw new NoOpenCoreException();
663 }
664 return currentCore.request(request);
665 }
666
677 if (currentCore == null) {
678 throw new NoOpenCoreException();
679 }
680 try {
681 return currentCore.queryNumIndexedFiles();
682 } catch (SolrServerException ex) {
683 throw new KeywordSearchModuleException(
684 NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
685 }
686
687
688 }
689
699 if (currentCore == null) {
700 throw new NoOpenCoreException();
701 }
702 try {
703 return currentCore.queryNumIndexedChunks();
704 } catch (SolrServerException ex) {
705 throw new KeywordSearchModuleException(
706 NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
707 }
708 }
709
719 if (currentCore == null) {
720 throw new NoOpenCoreException();
721 }
722 try {
723 return currentCore.queryNumIndexedDocuments();
724 } catch (SolrServerException ex) {
725 throw new KeywordSearchModuleException(
726 NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
727 }
728 }
729
738 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
739 if (currentCore == null) {
740 throw new NoOpenCoreException();
741 }
742 try {
743 return currentCore.queryIsIndexed(contentID);
744 } catch (SolrServerException ex) {
745 throw new KeywordSearchModuleException(
746 NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
747 }
748 }
749
759 public int queryNumFileChunks(
long fileID)
throws KeywordSearchModuleException, NoOpenCoreException {
760 if (currentCore == null) {
761 throw new NoOpenCoreException();
762 }
763 try {
764 return currentCore.queryNumFileChunks(fileID);
765 } catch (SolrServerException ex) {
766 throw new KeywordSearchModuleException(
767 NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
768 }
769 }
770
779 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
780 if (currentCore == null) {
781 throw new NoOpenCoreException();
782 }
783 try {
784 return currentCore.query(sq);
785 } catch (SolrServerException ex) {
786 throw new KeywordSearchModuleException(
787 NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
788 }
789 }
790
800 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
801 if (currentCore == null) {
802 throw new NoOpenCoreException();
803 }
804 try {
805 return currentCore.query(sq, method);
806 } catch (SolrServerException ex) {
807 throw new KeywordSearchModuleException(
808 NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
809 }
810 }
811
820 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
821 if (currentCore == null) {
822 throw new NoOpenCoreException();
823 }
824 try {
825 return currentCore.queryTerms(sq);
826 } catch (SolrServerException ex) {
827 throw new KeywordSearchModuleException(
828 NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
829 }
830 }
831
840 if (currentCore == null) {
841 throw new NoOpenCoreException();
842 }
843 return currentCore.getSolrContent(content.getId(), 0);
844 }
845
856 if (currentCore == null) {
857 throw new NoOpenCoreException();
858 }
859 return currentCore.getSolrContent(content.getId(), chunkID);
860 }
861
868 String
getSolrContent(
final long objectID)
throws NoOpenCoreException {
869 if (currentCore == null) {
870 throw new NoOpenCoreException();
871 }
872 return currentCore.getSolrContent(objectID, 0);
873 }
874
882 String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
883 if (currentCore == null) {
884 throw new NoOpenCoreException();
885 }
886 return currentCore.getSolrContent(objectID, chunkID);
887
888 }
895 return Ingester.getDefault();
896 }
897
908 }
909
917 private Core
openCore(String coreName,
File dataDir)
throws KeywordSearchModuleException {
918 try {
919 if (!dataDir.exists()) {
920 dataDir.mkdirs();
921 }
922
923 //handle a possible scenario when server process might not be fully started
924 if (!this.isRunning()) {
925 logger.log(Level.WARNING, "Core open requested, but server not yet running"); //NON-NLS
926 throw new KeywordSearchModuleException(
927 NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
928 }
929
930 CoreAdminRequest.Create createCore = new CoreAdminRequest.Create();
931 createCore.setDataDir(dataDir.getAbsolutePath());
932 createCore.setInstanceDir(instanceDir);
933 createCore.setCoreName(coreName);
934
935 this.solrServer.request(createCore);
936
937 final Core newCore = new Core(coreName);
938
939 return newCore;
940
941 } catch (SolrServerException ex) {
942 throw new KeywordSearchModuleException(
943 NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
944 } catch (IOException ex) {
945 throw new KeywordSearchModuleException(
946 NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg2"), ex);
947 }
948 }
949
950 class Core {
951
952 // handle to the core in Solr
953 private String name;
954 // the server to access a core needs to be built from a URL with the
955 // core in it, and is only good for core-specific operations
956 private HttpSolrServer solrCore;
957
958 private Core(String name) {
959 this.name = name;
960
961 this.solrCore = new HttpSolrServer(solrUrl + "/" + name);
962
963 //TODO test these settings
964 //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
965 //solrCore.setConnectionTimeout(1000);
966 solrCore.setDefaultMaxConnectionsPerHost(2);
967 solrCore.setMaxTotalConnections(5);
968 solrCore.setFollowRedirects(false); // defaults to false
969 // allowCompression defaults to false.
970 // Server side must support gzip or deflate for this to have any effect.
971 solrCore.setAllowCompression(true);
972 solrCore.setMaxRetries(1); // defaults to 0. > 1 not recommended.
973 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
974
975
976 }
977
978 private QueryResponse
query(SolrQuery sq)
throws SolrServerException {
979 return solrCore.query(sq);
980 }
981
982 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
983 try {
984 return solrCore.request(request);
985 } catch (IOException e) {
986 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
987 throw new SolrServerException(
988 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
989 }
990
991 }
992
993 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException {
994 return solrCore.query(sq, method);
995 }
996
997 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException {
998 QueryResponse qres = solrCore.query(sq);
999 return qres.getTermsResponse();
1000 }
1001
1002 private void commit() throws SolrServerException {
1003 try {
1004 //commit and block
1005 solrCore.commit(true, true);
1006 } catch (IOException e) {
1007 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1008 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1009 }
1010 }
1011
1012 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1013 try {
1014 solrCore.add(doc);
1015 } catch (SolrServerException ex) {
1016 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1017 throw new KeywordSearchModuleException(
1018 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1019 } catch (IOException ex) {
1020 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1021 throw new KeywordSearchModuleException(
1022 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1023 }
1024 }
1025
1033 final SolrQuery q = new SolrQuery();
1034 q.setQuery("*:*");
1035 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1036 if (chunkID != 0) {
1037 filterQuery = filterQuery + Server.ID_CHUNK_SEP + chunkID;
1038 }
1039 q.addFilterQuery(filterQuery);
1040 q.setFields(Schema.TEXT.toString());
1041 try {
1042 // Get the first result.
1043 SolrDocument solrDocument = solrCore.query(q).getResults().get(0);
1044 if (solrDocument != null) {
1045 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1046 if (fieldValues.size() == 1)
1047 // The indexed text field for artifacts will only have a single value.
1048 return fieldValues.toArray(new String[0])[0];
1049 else
1050 // The indexed text for files has 2 values, the file name and the file content.
1051 // We return the file content value.
1052 return fieldValues.toArray(new String[0])[1];
1053 }
1054 } catch (SolrServerException ex) {
1055 logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1056 return null;
1057 }
1058
1059 return null;
1060 }
1061
1062 synchronized void close() throws KeywordSearchModuleException {
1063 try {
1064 CoreAdminRequest.unloadCore(this.name, solrServer);
1065 } catch (SolrServerException ex) {
1066 throw new KeywordSearchModuleException(
1067 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1068 } catch (IOException ex) {
1069 throw new KeywordSearchModuleException(
1070 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1071 }
1072 }
1073
1084 }
1085
1094 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.ID_CHUNK_SEP + "*");
1095 q.setRows(0);
1096 int numChunks = (int)
query(q).getResults().getNumFound();
1097 return numChunks;
1098 }
1099
1110 SolrQuery q = new SolrQuery("*:*");
1111 q.setRows(0);
1112 return (
int)
query(q).getResults().getNumFound();
1113 }
1114
1122 private boolean queryIsIndexed(
long contentID)
throws SolrServerException {
1123 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1124 SolrQuery q = new SolrQuery("*:*");
1125 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1126 //q.setFields(Server.Schema.ID.toString());
1127 q.setRows(0);
1128 return (
int)
query(q).getResults().getNumFound() != 0;
1129 }
1130
1141 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1142 final SolrQuery q =
1143 new SolrQuery(Server.Schema.ID + ":" + id + Server.ID_CHUNK_SEP + "*");
1144 q.setRows(0);
1145 return (
int)
query(q).getResults().getNumFound();
1146 }
1147 }
1148
1149 class ServerAction extends AbstractAction {
1150
1151 @Override
1152 public void actionPerformed(ActionEvent e) {
1153 logger.log(Level.INFO, e.paramString().trim());
1154 }
1155 }
1156
1160 class SolrServerNoPortException extends SocketException {
1161
1165 private int port;
1166
1167 SolrServerNoPortException(int port) {
1168 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1169 Server.PROPERTIES_CURRENT_SERVER_PORT));
1170 this.port = port;
1171 }
1172
1173 int getPortNumber() {
1174 return port;
1175 }
1176 }
1177 }
int queryNumIndexedFiles()
synchronized Content getParent()
ServerAction serverAction
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
String getCaseDirectory()
static final Logger logger
void validateIndexLocation(Case theCase)
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
synchronized Core openCore(Case theCase)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
static final String DEFAULT_CORE_NAME
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
Core openCore(String coreName, File dataDir)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
static Case getCurrentCase()
volatile Core currentCore
static final int MAX_SOLR_MEM_MB
static Ingester getIngester()
static String getChunkIdString(long parentID, int childID)
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static Logger getLogger(String name)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)