1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.List;
43 import java.util.concurrent.locks.ReentrantReadWriteLock;
44 import java.util.logging.Level;
45 import javax.swing.AbstractAction;
46 import org.apache.solr.client.solrj.SolrQuery;
47 import org.apache.solr.client.solrj.SolrRequest;
48 import org.apache.solr.client.solrj.SolrServerException;
49 import org.apache.solr.client.solrj.impl.HttpSolrServer;
50 import org.apache.solr.client.solrj.impl.XMLResponseParser;
51 import org.apache.solr.client.solrj.request.CoreAdminRequest;
52 import org.apache.solr.client.solrj.response.CoreAdminResponse;
53 import org.apache.solr.client.solrj.response.QueryResponse;
54 import org.apache.solr.client.solrj.response.TermsResponse;
55 import org.apache.solr.common.SolrDocument;
56 import org.apache.solr.common.SolrDocumentList;
57 import org.apache.solr.common.SolrException;
58 import org.apache.solr.common.SolrInputDocument;
59 import org.apache.solr.common.util.NamedList;
60 import org.openide.modules.InstalledFileLocator;
61 import org.openide.modules.Places;
62 import org.openide.util.NbBundle;
71
77
82
83 ID {
84 @Override
85 public String toString() {
86 return "id"; //NON-NLS
87 }
88 },
89 IMAGE_ID {
90 @Override
91 public String toString() {
92 return "image_id"; //NON-NLS
93 }
94 },
95 // This is not stored or index . it is copied to Text and Content_Ws
96 CONTENT {
97 @Override
98 public String toString() {
99 return "content"; //NON-NLS
100 }
101 },
102 CONTENT_STR {
103 @Override
104 public String toString() {
105 return "content_str"; //NON-NLS
106 }
107 },
108 TEXT {
109 @Override
110 public String toString() {
111 return "text"; //NON-NLS
112 }
113 },
114 CONTENT_WS {
115 @Override
116 public String toString() {
117 return "content_ws"; //NON-NLS
118 }
119 },
120 FILE_NAME {
121 @Override
122 public String toString() {
123 return "file_name"; //NON-NLS
124 }
125 },
126 // note that we no longer index this field
127 CTIME {
128 @Override
129 public String toString() {
130 return "ctime"; //NON-NLS
131 }
132 },
133 // note that we no longer index this field
134 ATIME {
135 @Override
136 public String toString() {
137 return "atime"; //NON-NLS
138 }
139 },
140 // note that we no longer index this field
141 MTIME {
142 @Override
143 public String toString() {
144 return "mtime"; //NON-NLS
145 }
146 },
147 // note that we no longer index this field
148 CRTIME {
149 @Override
150 public String toString() {
151 return "crtime"; //NON-NLS
152 }
153 },
154 NUM_CHUNKS {
155 @Override
156 public String toString() {
157 return "num_chunks"; //NON-NLS
158 }
159 },
160 CHUNK_SIZE {
161 @Override
162 public String toString() {
163 return "chunk_size"; //NON-NLS
164 }
165 }
166 };
167
168 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
169 //max content size we can send to Solr
172 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
173 @Deprecated
179 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
180 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
181 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
182 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
183 private static final String
KEY =
"jjk#09s";
//NON-NLS
184 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
185 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
186 static final int DEFAULT_SOLR_STOP_PORT = 34343;
189 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
190 private static final String
SOLR =
"solr";
192
194
197
198 // A reference to the locally running Solr instance.
200
201 // A reference to the Solr server we are currently connected to for the Case.
202 // This could be a local or remote server.
204
207
212
219
220 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
221 serverAction = new ServerAction();
222 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
224
226 if (!solrHome.toFile().exists()) {
227 try {
228 Files.createDirectory(solrHome);
229 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
230 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
231 } catch (IOException ex) {
232 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
233 }
234 }
235 currentCoreLock = new ReentrantReadWriteLock(true);
236
237 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
238 }
239
241
243 try {
245 } catch (NumberFormatException nfe) {
246 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
247 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
248 }
249 } else {
250 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
252 }
253
255 try {
257 } catch (NumberFormatException nfe) {
258 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
259 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
260 }
261 } else {
262 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
264 }
265 }
266
267 @Override
268 public void finalize() throws java.lang.Throwable {
269 stop();
270 super.finalize();
271 }
272
274 serverAction.addPropertyChangeListener(l);
275 }
276
277 int getCurrentSolrServerPort() {
279 }
280
281 int getCurrentSolrStopPort() {
283 }
284
289
290 InputStream stream;
291 OutputStream out;
292 volatile boolean doRun = true;
293
295 this.stream = stream;
296 try {
297 final String log = Places.getUserDirectory().getAbsolutePath()
298 + File.separator + "var" + File.separator + "log" //NON-NLS
299 + File.separator + "solr.log." + type; //NON-NLS
300 File outputFile = new File(log.concat(".0"));
301 File first = new File(log.concat(".1"));
302 File second = new File(log.concat(".2"));
303 if (second.exists()) {
304 second.delete();
305 }
306 if (first.exists()) {
307 first.renameTo(second);
308 }
309 if (outputFile.exists()) {
310 outputFile.renameTo(first);
311 } else {
312 outputFile.createNewFile();
313 }
314 out = new FileOutputStream(outputFile);
315
316 } catch (Exception ex) {
317 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
318 }
319 }
320
321 void stopRun() {
322 doRun = false;
323 }
324
325 @Override
327
328 try (InputStreamReader isr = new InputStreamReader(stream);
329 BufferedReader br = new BufferedReader(isr);
331 BufferedWriter bw = new BufferedWriter(osw);) {
332
333 String line = null;
334 while (doRun && (line = br.readLine()) != null) {
335 bw.write(line);
336 bw.newLine();
337 if (DEBUG) {
338 //flush buffers if dev version for debugging
339 bw.flush();
340 }
341 }
342 bw.flush();
343 } catch (IOException ex) {
344 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
345 }
346 }
347 }
348
359 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
360 List<String> commandLine = new ArrayList<>();
361 commandLine.add(javaPath);
362 commandLine.add(MAX_SOLR_MEM_MB_PAR);
363 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
364 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
365 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
366 commandLine.add("-jar"); //NON-NLS
367 commandLine.add("start.jar"); //NON-NLS
368
369 commandLine.addAll(solrArguments);
370
371 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
372 solrProcessBuilder.directory(solrFolder);
373
374 // Redirect stdout and stderr to files to prevent blocking.
375 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
376 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
377
378 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
379 solrProcessBuilder.redirectError(solrStderrPath.toFile());
380
381 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
382 Process process = solrProcessBuilder.start();
383 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
384 return process;
385 }
386
392 List<Long> getSolrPIDs() {
393 List<Long> pids = new ArrayList<>();
394
395 //NOTE: these needs to be in sync with process start string in start()
396 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
397
399 if (pidsArr != null) {
400 for (int i = 0; i < pidsArr.length; ++i) {
401 pids.add(pidsArr[i]);
402 }
403 }
404
405 return pids;
406 }
407
412 void killSolr() {
413 List<Long> solrPids = getSolrPIDs();
414 for (long pid : solrPids) {
415 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
416 PlatformUtil.killProcess(pid);
417 }
418 }
419
425 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
426 if (isRunning()) {
427 // If a Solr server is running we stop it.
428 stop();
429 }
430
431 if (!isPortAvailable(currentSolrServerPort)) {
432 // There is something already listening on our port. Let's see if
433 // this is from an earlier run that didn't successfully shut down
434 // and if so kill it.
435 final List<Long> pids = this.getSolrPIDs();
436
437 // If the culprit listening on the port is not a Solr process
438 // we refuse to start.
439 if (pids.isEmpty()) {
440 throw new SolrServerNoPortException(currentSolrServerPort);
441 }
442
443 // Ok, we've tried to stop it above but there still appears to be
444 // a Solr process listening on our port so we forcefully kill it.
445 killSolr();
446
447 // If either of the ports are still in use after our attempt to kill
448 // previously running processes we give up and throw an exception.
449 if (!isPortAvailable(currentSolrServerPort)) {
450 throw new SolrServerNoPortException(currentSolrServerPort);
451 }
452 if (!isPortAvailable(currentSolrStopPort)) {
453 throw new SolrServerNoPortException(currentSolrStopPort);
454 }
455 }
456
457 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
458
459 if (isPortAvailable(currentSolrServerPort)) {
460 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
461 try {
463 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
464 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
465
466 try {
467 //block for 10 seconds, give time to fully start the process
468 //so if it's restarted solr operations can be resumed seamlessly
469 Thread.sleep(10 * 1000);
470 } catch (InterruptedException ex) {
471 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
472 }
473
474 final List<Long> pids = this.getSolrPIDs();
475 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
476 } catch (SecurityException ex) {
477 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
478 throw new KeywordSearchModuleException(
479 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
480 } catch (IOException ex) {
481 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
482 throw new KeywordSearchModuleException(
483 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
484 }
485 }
486 }
487
493 static boolean isPortAvailable(int port) {
494 ServerSocket ss = null;
495 try {
496
497 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
498 if (ss.isBound()) {
499 ss.setReuseAddress(true);
500 ss.close();
501 return true;
502 }
503
504 } catch (IOException e) {
505 } finally {
506 if (ss != null) {
507 try {
508 ss.close();
509 } catch (IOException e) {
510 /*
511 * should not be thrown
512 */
513 }
514 }
515 }
516 return false;
517 }
518
524 void changeSolrServerPort(int port) {
525 currentSolrServerPort = port;
526 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
527 }
528
534 void changeSolrStopPort(int port) {
535 currentSolrStopPort = port;
536 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
537 }
538
544 synchronized void stop() {
545
546 try {
547 // Close any open core before stopping server
548 closeCore();
549 } catch (KeywordSearchModuleException e) {
550 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
551 }
552
553 try {
554 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
555
556 //try graceful shutdown
557 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
558
559 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
560 process.waitFor();
561
562 //if still running, forcefully stop it
563 if (curSolrProcess != null) {
564 curSolrProcess.destroy();
565 curSolrProcess = null;
566 }
567
568 } catch (IOException | InterruptedException ex) {
569 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
570 } finally {
571 //stop Solr stream -> log redirect threads
572 try {
573 if (errorRedirectThread != null) {
574 errorRedirectThread.stopRun();
575 errorRedirectThread = null;
576 }
577 } finally {
578 //if still running, kill it
579 killSolr();
580 }
581
582 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
583 }
584 }
585
593 synchronized boolean isRunning() throws KeywordSearchModuleException {
594 try {
595
596 if (isPortAvailable(currentSolrServerPort)) {
597 return false;
598 }
599
600 // making a status request here instead of just doing solrServer.ping(), because
601 // that doesn't work when there are no cores
602 //TODO handle timeout in cases when some other type of server on that port
603 CoreAdminRequest.getStatus(null, localSolrServer);
604
605 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
606 } catch (SolrServerException ex) {
607
608 Throwable cause = ex.getRootCause();
609
610 // TODO: check if SocketExceptions should actually happen (is
611 // probably caused by starting a connection as the server finishes
612 // shutting down)
613 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
614 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
615 return false;
616 } else {
617 throw new KeywordSearchModuleException(
618 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
619 }
620 } catch (SolrException ex) {
621 // Just log 404 errors for now...
622 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
623 return false;
624 } catch (IOException ex) {
625 throw new KeywordSearchModuleException(
626 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
627 }
628
629 return true;
630 }
631
632 /*
633 * ** Convenience methods for use while we only open one case at a time ***
634 */
644 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
645 currentCoreLock.writeLock().lock();
646 try {
647 currentCore =
openCore(theCase, index);
648
649 try {
650 // execute a test query. if it fails, an exception will be thrown
652 } catch (NoOpenCoreException ex) {
653 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
654 }
655
656 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
657 } finally {
658 currentCoreLock.writeLock().unlock();
659 }
660 }
661
667 boolean coreIsOpen() {
668 currentCoreLock.readLock().lock();
669 try {
670 return (null != currentCore);
671 } finally {
672 currentCoreLock.readLock().unlock();
673 }
674 }
675
676 Index getIndexInfo() throws NoOpenCoreException {
677 currentCoreLock.readLock().lock();
678 try {
679 if (null == currentCore) {
680 throw new NoOpenCoreException();
681 }
682 return currentCore.getIndexInfo();
683 } finally {
684 currentCoreLock.readLock().unlock();
685 }
686 }
687
688 void closeCore() throws KeywordSearchModuleException {
689 currentCoreLock.writeLock().lock();
690 try {
691 if (null != currentCore) {
692 currentCore.close();
693 currentCore = null;
694 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
695 }
696 } finally {
697 currentCoreLock.writeLock().unlock();
698 }
699 }
700
701 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
702 currentCoreLock.readLock().lock();
703 try {
704 if (null == currentCore) {
705 throw new NoOpenCoreException();
706 }
707 currentCore.addDocument(doc);
708 } finally {
709 currentCoreLock.readLock().unlock();
710 }
711 }
712
721 @NbBundle.Messages({
722 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
723 void deleteCore(String coreName, Case.CaseType caseType) throws KeywordSearchServiceException {
724 try {
725 HttpSolrServer solrServer;
726 if (caseType == CaseType.SINGLE_USER_CASE) {
727 Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
728 solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
729 } else {
730 String host = UserPreferences.getIndexingServerHost();
731 String port = UserPreferences.getIndexingServerPort();
732 solrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
733 }
734 connectToSolrServer(solrServer);
735 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
736 if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
737 /*
738 * Send a core unload request to the Solr server, with the
739 * parameter set that request deleting the index and the
740 * instance directory (deleteInstanceDir = true). Note that this
741 * removes everything related to the core on the server (the
742 * index directory, the configuration files, etc.), but does not
743 * delete the actual Solr text index because it is currently
744 * stored in the case directory.
745 */
746 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
747 }
748 } catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
749 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
750 }
751 }
752
764 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
765
766 try {
769 } else {
772 currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
773 }
774 connectToSolrServer(currentSolrServer);
775
776 } catch (SolrServerException | IOException ex) {
777 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
778 }
779
780 try {
781 File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
782 if (!dataDir.exists()) {
783 dataDir.mkdirs();
784 }
785
786 if (!this.isRunning()) {
787 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
788 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
789 }
790
791 String coreName = index.getIndexName();
793 /*
794 * The core either does not exist or it is not loaded. Make a
795 * request that will cause the core to be created if it does not
796 * exist or loaded if it already exists.
797 */
798
799 // In single user mode, if there is a core.properties file already,
800 // we've hit a solr bug. Compensate by deleting it.
802 Path corePropertiesFile = Paths.get(solrFolder.toString(),
SOLR, coreName,
CORE_PROPERTIES);
803 if (corePropertiesFile.toFile().exists()) {
804 try {
805 corePropertiesFile.toFile().delete();
806 } catch (Exception ex) {
807 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
808 }
809 }
810 }
811
812 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
813 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
814 createCoreRequest.setCoreName(coreName);
815 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
816 createCoreRequest.setIsLoadOnStartup(false);
817 createCoreRequest.setIsTransient(true);
818 currentSolrServer.request(createCoreRequest);
819 }
820
822 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
823 }
824
825 return new Core(coreName, theCase.getCaseType(), index);
826
827 } catch (Exception ex) {
828 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
829 }
830 }
831
837 void commit() throws SolrServerException, NoOpenCoreException {
838 currentCoreLock.readLock().lock();
839 try {
840 if (null == currentCore) {
841 throw new NoOpenCoreException();
842 }
843 currentCore.commit();
844 } finally {
845 currentCoreLock.readLock().unlock();
846 }
847 }
848
849 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
850 currentCoreLock.readLock().lock();
851 try {
852 if (null == currentCore) {
853 throw new NoOpenCoreException();
854 }
855 return currentCore.request(request);
856 } finally {
857 currentCoreLock.readLock().unlock();
858 }
859 }
860
872 currentCoreLock.readLock().lock();
873 try {
874 if (null == currentCore) {
875 throw new NoOpenCoreException();
876 }
877 try {
878 return currentCore.queryNumIndexedFiles();
879 } catch (SolrServerException | IOException ex) {
880 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
881 }
882 } finally {
883 currentCoreLock.readLock().unlock();
884 }
885 }
886
897 currentCoreLock.readLock().lock();
898 try {
899 if (null == currentCore) {
900 throw new NoOpenCoreException();
901 }
902 try {
903 return currentCore.queryNumIndexedChunks();
904 } catch (SolrServerException | IOException ex) {
905 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
906 }
907 } finally {
908 currentCoreLock.readLock().unlock();
909 }
910 }
911
922 currentCoreLock.readLock().lock();
923 try {
924 if (null == currentCore) {
925 throw new NoOpenCoreException();
926 }
927 try {
928 return currentCore.queryNumIndexedDocuments();
929 } catch (SolrServerException | IOException ex) {
930 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
931 }
932 } finally {
933 currentCoreLock.readLock().unlock();
934 }
935 }
936
947 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
948 currentCoreLock.readLock().lock();
949 try {
950 if (null == currentCore) {
951 throw new NoOpenCoreException();
952 }
953 try {
954 return currentCore.queryIsIndexed(contentID);
955 } catch (SolrServerException | IOException ex) {
956 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
957 }
958
959 } finally {
960 currentCoreLock.readLock().unlock();
961 }
962 }
963
975 public int queryNumFileChunks(
long fileID)
throws KeywordSearchModuleException, NoOpenCoreException {
976 currentCoreLock.readLock().lock();
977 try {
978 if (null == currentCore) {
979 throw new NoOpenCoreException();
980 }
981 try {
982 return currentCore.queryNumFileChunks(fileID);
983 } catch (SolrServerException | IOException ex) {
984 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
985 }
986 } finally {
987 currentCoreLock.readLock().unlock();
988 }
989 }
990
1001 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1002 currentCoreLock.readLock().lock();
1003 try {
1004 if (null == currentCore) {
1005 throw new NoOpenCoreException();
1006 }
1007 try {
1008 return currentCore.query(sq);
1009 } catch (SolrServerException ex) {
1010 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1011 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1012 }
1013 } finally {
1014 currentCoreLock.readLock().unlock();
1015 }
1016 }
1017
1029 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1030 currentCoreLock.readLock().lock();
1031 try {
1032 if (null == currentCore) {
1033 throw new NoOpenCoreException();
1034 }
1035 try {
1036 return currentCore.query(sq, method);
1037 } catch (SolrServerException | IOException ex) {
1038 logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1039 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1040 }
1041 } finally {
1042 currentCoreLock.readLock().unlock();
1043 }
1044 }
1045
1056 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1057 currentCoreLock.readLock().lock();
1058 try {
1059 if (null == currentCore) {
1060 throw new NoOpenCoreException();
1061 }
1062 try {
1063 return currentCore.queryTerms(sq);
1064 } catch (SolrServerException | IOException ex) {
1065 logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1066 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1067 }
1068 } finally {
1069 currentCoreLock.readLock().unlock();
1070 }
1071 }
1072
1083 currentCoreLock.readLock().lock();
1084 try {
1085 if (null == currentCore) {
1086 throw new NoOpenCoreException();
1087 }
1088 return currentCore.getSolrContent(content.getId(), 0);
1089 } finally {
1090 currentCoreLock.readLock().unlock();
1091 }
1092 }
1093
1106 public String
getSolrContent(
final Content content,
int chunkID)
throws NoOpenCoreException {
1107 currentCoreLock.readLock().lock();
1108 try {
1109 if (null == currentCore) {
1110 throw new NoOpenCoreException();
1111 }
1112 return currentCore.getSolrContent(content.getId(), chunkID);
1113 } finally {
1114 currentCoreLock.readLock().unlock();
1115 }
1116 }
1117
1128 currentCoreLock.readLock().lock();
1129 try {
1130 if (null == currentCore) {
1131 throw new NoOpenCoreException();
1132 }
1133 return currentCore.getSolrContent(objectID, 0);
1134 } finally {
1135 currentCoreLock.readLock().unlock();
1136 }
1137 }
1138
1149 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1150 currentCoreLock.readLock().lock();
1151 try {
1152 if (null == currentCore) {
1153 throw new NoOpenCoreException();
1154 }
1155 return currentCore.getSolrContent(objectID, chunkID);
1156 } finally {
1157 currentCoreLock.readLock().unlock();
1158 }
1159 }
1160
1172 }
1173
1182 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1183 CoreAdminRequest.getStatus(null, solrServer);
1184 }
1185
1199 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1200 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1201 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1202 }
1203
1215 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1216 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1217 if (null != dataDirPath) {
1218 File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1219 return indexDir.exists();
1220 } else {
1221 return false;
1222 }
1223 }
1224
1225 class Core {
1226
1227 // handle to the core in Solr
1228 private final String name;
1229
1231
1232 private final Index textIndex;
1233
1234 // the server to access a core needs to be built from a URL with the
1235 // core in it, and is only good for core-specific operations
1236 private final HttpSolrServer solrCore;
1237
1238 private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
1239
1240 private Core(String name,
CaseType caseType, Index index) {
1241 this.name = name;
1242 this.caseType = caseType;
1243 this.textIndex = index;
1244
1245 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1246
1247 //TODO test these settings
1248 // socket read timeout, make large enough so can index larger files
1249 solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
1250 //solrCore.setConnectionTimeout(1000);
1251 solrCore.setDefaultMaxConnectionsPerHost(32);
1252 solrCore.setMaxTotalConnections(32);
1253 solrCore.setFollowRedirects(false); // defaults to false
1254 // allowCompression defaults to false.
1255 // Server side must support gzip or deflate for this to have any effect.
1256 solrCore.setAllowCompression(true);
1257 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1258
1259 }
1260
1266 String getName() {
1267 return name;
1268 }
1269
1270 private Index getIndexInfo() {
1271 return this.textIndex;
1272 }
1273
1274 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1275 return solrCore.query(sq);
1276 }
1277
1278 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1279 try {
1280 return solrCore.request(request);
1281 } catch (IOException e) {
1282 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1283 throw new SolrServerException(
1284 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1285 }
1286
1287 }
1288
1289 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1290 return solrCore.query(sq, method);
1291 }
1292
1293 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1294 QueryResponse qres = solrCore.query(sq);
1295 return qres.getTermsResponse();
1296 }
1297
1298 private void commit() throws SolrServerException {
1299 try {
1300 //commit and block
1301 solrCore.commit(true, true);
1302 } catch (IOException e) {
1303 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1304 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1305 }
1306 }
1307
1308 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1309 try {
1310 solrCore.add(doc);
1311 } catch (SolrServerException ex) {
1312 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1313 throw new KeywordSearchModuleException(
1314 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1315 } catch (IOException ex) {
1316 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1317 throw new KeywordSearchModuleException(
1318 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1319 }
1320 }
1321
1332 final SolrQuery q = new SolrQuery();
1333 q.setQuery("*:*");
1334 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1335 if (chunkID != 0) {
1336 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1337 }
1338 q.addFilterQuery(filterQuery);
1339 q.setFields(Schema.TEXT.toString());
1340 try {
1341 // Get the first result.
1342 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1343
1344 if (!solrDocuments.isEmpty()) {
1345 SolrDocument solrDocument = solrDocuments.get(0);
1346 if (solrDocument != null) {
1347 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1348 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1349 {
1350 return fieldValues.toArray(new String[0])[0];
1351 } else // The indexed text for files has 2 values, the file name and the file content.
1352 // We return the file content value.
1353 {
1354 return fieldValues.toArray(new String[0])[1];
1355 }
1356 }
1357 }
1358 } catch (SolrServerException ex) {
1359 logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
1360 return null;
1361 }
1362
1363 return null;
1364 }
1365
1366 synchronized void close() throws KeywordSearchModuleException {
1367 // We only unload cores for "single-user" cases.
1369 return;
1370 }
1371
1372 try {
1373 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1374 } catch (SolrServerException ex) {
1375 throw new KeywordSearchModuleException(
1376 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1377 } catch (IOException ex) {
1378 throw new KeywordSearchModuleException(
1379 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1380 }
1381 }
1382
1394 }
1395
1406 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1407 q.setRows(0);
1408 int numChunks = (int)
query(q).getResults().getNumFound();
1409 return numChunks;
1410 }
1411
1423 SolrQuery q = new SolrQuery("*:*");
1424 q.setRows(0);
1425 return (
int)
query(q).getResults().getNumFound();
1426 }
1427
1437 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1438 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1439 SolrQuery q = new SolrQuery("*:*");
1440 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1441 //q.setFields(Server.Schema.ID.toString());
1442 q.setRows(0);
1443 return (
int)
query(q).getResults().getNumFound() != 0;
1444 }
1445
1457 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1458 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1459 final SolrQuery q
1460 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1461 q.setRows(0);
1462 return (
int)
query(q).getResults().getNumFound();
1463 }
1464 }
1465
1466 class ServerAction extends AbstractAction {
1467
1468 private static final long serialVersionUID = 1L;
1469
1470 @Override
1471 public void actionPerformed(ActionEvent e) {
1472 logger.log(Level.INFO, e.paramString().trim());
1473 }
1474 }
1475
1479 class SolrServerNoPortException extends SocketException {
1480
1481 private static final long serialVersionUID = 1L;
1482
1486 private final int port;
1487
1488 SolrServerNoPortException(int port) {
1489 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1490 Server.PROPERTIES_CURRENT_SERVER_PORT));
1491 this.port = port;
1492 }
1493
1494 int getPortNumber() {
1495 return port;
1496 }
1497 }
1498 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static final int MAX_SOLR_MEM_MB
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)