1 /*
2 * Autopsy Forensic Browser
3 *
4 * Copyright 2011-2016 Basis Technology Corp.
5 * Contact: carrier <at> sleuthkit <dot> org
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19 package org.sleuthkit.autopsy.keywordsearch;
20
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.List;
43 import java.util.concurrent.locks.ReentrantReadWriteLock;
44 import java.util.logging.Level;
45 import javax.swing.AbstractAction;
46 import org.apache.solr.client.solrj.SolrQuery;
47 import org.apache.solr.client.solrj.SolrRequest;
48 import org.apache.solr.client.solrj.SolrServerException;
49 import org.apache.solr.client.solrj.impl.HttpSolrServer;
50 import org.apache.solr.client.solrj.impl.XMLResponseParser;
51 import org.apache.solr.client.solrj.request.CoreAdminRequest;
52 import org.apache.solr.client.solrj.response.CoreAdminResponse;
53 import org.apache.solr.client.solrj.response.QueryResponse;
54 import org.apache.solr.client.solrj.response.TermsResponse;
55 import org.apache.solr.common.SolrDocument;
56 import org.apache.solr.common.SolrDocumentList;
57 import org.apache.solr.common.SolrException;
58 import org.apache.solr.common.SolrInputDocument;
59 import org.apache.solr.common.util.NamedList;
60 import org.openide.modules.InstalledFileLocator;
61 import org.openide.modules.Places;
62 import org.openide.util.NbBundle;
71
77
82
83 ID {
84 @Override
85 public String toString() {
86 return "id"; //NON-NLS
87 }
88 },
89 IMAGE_ID {
90 @Override
91 public String toString() {
92 return "image_id"; //NON-NLS
93 }
94 },
95 // This is not stored or index . it is copied to Text and Content_Ws
96 CONTENT {
97 @Override
98 public String toString() {
99 return "content"; //NON-NLS
100 }
101 },
102 CONTENT_STR {
103 @Override
104 public String toString() {
105 return "content_str"; //NON-NLS
106 }
107 },
108 TEXT {
109 @Override
110 public String toString() {
111 return "text"; //NON-NLS
112 }
113 },
114 CONTENT_WS {
115 @Override
116 public String toString() {
117 return "content_ws"; //NON-NLS
118 }
119 },
120 FILE_NAME {
121 @Override
122 public String toString() {
123 return "file_name"; //NON-NLS
124 }
125 },
126 // note that we no longer index this field
127 CTIME {
128 @Override
129 public String toString() {
130 return "ctime"; //NON-NLS
131 }
132 },
133 // note that we no longer index this field
134 ATIME {
135 @Override
136 public String toString() {
137 return "atime"; //NON-NLS
138 }
139 },
140 // note that we no longer index this field
141 MTIME {
142 @Override
143 public String toString() {
144 return "mtime"; //NON-NLS
145 }
146 },
147 // note that we no longer index this field
148 CRTIME {
149 @Override
150 public String toString() {
151 return "crtime"; //NON-NLS
152 }
153 },
154 NUM_CHUNKS {
155 @Override
156 public String toString() {
157 return "num_chunks"; //NON-NLS
158 }
159 },
160 CHUNK_SIZE {
161 @Override
162 public String toString() {
163 return "chunk_size"; //NON-NLS
164 }
165 }
166 };
167
168 public static final String
HL_ANALYZE_CHARS_UNLIMITED =
"500000";
//max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
169 //max content size we can send to Solr
172 public static final String
CORE_EVT =
"CORE_EVT";
//NON-NLS
173 @Deprecated
179 private static final int MAX_SOLR_MEM_MB = 512;
//TODO set dynamically based on avail. system resources
180 static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
181 static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
182 static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
183 private static final String
KEY =
"jjk#09s";
//NON-NLS
184 static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
185 static final int DEFAULT_SOLR_SERVER_PORT = 23232;
186 static final int DEFAULT_SOLR_STOP_PORT = 34343;
189 private static final boolean DEBUG =
false;
//(Version.getBuildType() == Version.Type.DEVELOPMENT);
190 private static final String
SOLR =
"solr";
192
194
197
198 // A reference to the locally running Solr instance.
200
201 // A reference to the Solr server we are currently connected to for the Case.
202 // This could be a local or remote server.
204
207
212
219
220 this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
221 serverAction = new ServerAction();
222 solrFolder = InstalledFileLocator.getDefault().locate(
"solr",
Server.class.getPackage().getName(),
false);
//NON-NLS
224
226 if (!solrHome.toFile().exists()) {
227 try {
228 Files.createDirectory(solrHome);
229 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
230 Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
231 } catch (IOException ex) {
232 logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
233 }
234 }
235 currentCoreLock = new ReentrantReadWriteLock(true);
236
237 logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
238 }
239
241
243 try {
245 } catch (NumberFormatException nfe) {
246 logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
247 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
248 }
249 } else {
250 currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
252 }
253
255 try {
257 } catch (NumberFormatException nfe) {
258 logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
259 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
260 }
261 } else {
262 currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
264 }
265 }
266
267 @Override
268 public void finalize() throws java.lang.Throwable {
269 stop();
270 super.finalize();
271 }
272
274 serverAction.addPropertyChangeListener(l);
275 }
276
277 int getCurrentSolrServerPort() {
279 }
280
281 int getCurrentSolrStopPort() {
283 }
284
289
290 InputStream stream;
291 OutputStream out;
292 volatile boolean doRun = true;
293
295 this.stream = stream;
296 try {
297 final String log = Places.getUserDirectory().getAbsolutePath()
298 +
File.separator +
"var" +
File.separator +
"log" //NON-NLS
299 +
File.separator +
"solr.log." + type;
//NON-NLS
300 File outputFile =
new File(log.concat(
".0"));
301 File first =
new File(log.concat(
".1"));
302 File second =
new File(log.concat(
".2"));
303 if (second.exists()) {
304 second.delete();
305 }
306 if (first.exists()) {
307 first.renameTo(second);
308 }
309 if (outputFile.
exists()) {
310 outputFile.renameTo(first);
311 } else {
312 outputFile.createNewFile();
313 }
314 out = new FileOutputStream(outputFile);
315
316 } catch (Exception ex) {
317 logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
318 }
319 }
320
321 void stopRun() {
322 doRun = false;
323 }
324
325 @Override
327
328 try (InputStreamReader isr = new InputStreamReader(stream);
329 BufferedReader br = new BufferedReader(isr);
331 BufferedWriter bw = new BufferedWriter(osw);) {
332
333 String line = null;
334 while (doRun && (line = br.readLine()) != null) {
335 bw.write(line);
336 bw.newLine();
337 if (DEBUG) {
338 //flush buffers if dev version for debugging
339 bw.flush();
340 }
341 }
342 bw.flush();
343 } catch (IOException ex) {
344 logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
345 }
346 }
347 }
348
359 final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
360 List<String> commandLine = new ArrayList<>();
361 commandLine.add(javaPath);
362 commandLine.add(MAX_SOLR_MEM_MB_PAR);
363 commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
364 commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
365 commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
366 commandLine.add("-jar"); //NON-NLS
367 commandLine.add("start.jar"); //NON-NLS
368
369 commandLine.addAll(solrArguments);
370
371 ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
372 solrProcessBuilder.directory(solrFolder);
373
374 // Redirect stdout and stderr to files to prevent blocking.
375 Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
376 solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
377
378 Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
379 solrProcessBuilder.redirectError(solrStderrPath.toFile());
380
381 logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
382 Process process = solrProcessBuilder.start();
383 logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
384 return process;
385 }
386
392 List<Long> getSolrPIDs() {
393 List<Long> pids = new ArrayList<>();
394
395 //NOTE: these needs to be in sync with process start string in start()
396 final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
397
399 if (pidsArr != null) {
400 for (int i = 0; i < pidsArr.length; ++i) {
401 pids.add(pidsArr[i]);
402 }
403 }
404
405 return pids;
406 }
407
412 void killSolr() {
413 List<Long> solrPids = getSolrPIDs();
414 for (long pid : solrPids) {
415 logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
416 PlatformUtil.killProcess(pid);
417 }
418 }
419
425 void start() throws KeywordSearchModuleException, SolrServerNoPortException {
426 if (isRunning()) {
427 // If a Solr server is running we stop it.
428 stop();
429 }
430
431 if (!isPortAvailable(currentSolrServerPort)) {
432 // There is something already listening on our port. Let's see if
433 // this is from an earlier run that didn't successfully shut down
434 // and if so kill it.
435 final List<Long> pids = this.getSolrPIDs();
436
437 // If the culprit listening on the port is not a Solr process
438 // we refuse to start.
439 if (pids.isEmpty()) {
440 throw new SolrServerNoPortException(currentSolrServerPort);
441 }
442
443 // Ok, we've tried to stop it above but there still appears to be
444 // a Solr process listening on our port so we forcefully kill it.
445 killSolr();
446
447 // If either of the ports are still in use after our attempt to kill
448 // previously running processes we give up and throw an exception.
449 if (!isPortAvailable(currentSolrServerPort)) {
450 throw new SolrServerNoPortException(currentSolrServerPort);
451 }
452 if (!isPortAvailable(currentSolrStopPort)) {
453 throw new SolrServerNoPortException(currentSolrStopPort);
454 }
455 }
456
457 logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
458
459 if (isPortAvailable(currentSolrServerPort)) {
460 logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
461 try {
463 Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
464 "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
465
466 try {
467 //block for 10 seconds, give time to fully start the process
468 //so if it's restarted solr operations can be resumed seamlessly
469 Thread.sleep(10 * 1000);
470 } catch (InterruptedException ex) {
471 logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
472 }
473
474 final List<Long> pids = this.getSolrPIDs();
475 logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
476 } catch (SecurityException ex) {
477 logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
478 throw new KeywordSearchModuleException(
479 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
480 } catch (IOException ex) {
481 logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
482 throw new KeywordSearchModuleException(
483 NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
484 }
485 }
486 }
487
493 static boolean isPortAvailable(int port) {
494 ServerSocket ss = null;
495 try {
496
497 ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
498 if (ss.isBound()) {
499 ss.setReuseAddress(true);
500 ss.close();
501 return true;
502 }
503
504 } catch (IOException e) {
505 } finally {
506 if (ss != null) {
507 try {
508 ss.close();
509 } catch (IOException e) {
510 /*
511 * should not be thrown
512 */
513 }
514 }
515 }
516 return false;
517 }
518
524 void changeSolrServerPort(int port) {
525 currentSolrServerPort = port;
526 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
527 }
528
534 void changeSolrStopPort(int port) {
535 currentSolrStopPort = port;
536 ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
537 }
538
544 synchronized void stop() {
545
546 try {
547 // Close any open core before stopping server
548 closeCore();
549 } catch (KeywordSearchModuleException e) {
550 logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
551 }
552
553 try {
554 logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
555
556 //try graceful shutdown
557 Process process =
runSolrCommand(
new ArrayList<>(Arrays.asList(
"--stop")));
//NON-NLS
558
559 logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
560 process.waitFor();
561
562 //if still running, forcefully stop it
563 if (curSolrProcess != null) {
564 curSolrProcess.destroy();
565 curSolrProcess = null;
566 }
567
568 } catch (IOException | InterruptedException ex) {
569 logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
570 } finally {
571 //stop Solr stream -> log redirect threads
572 try {
573 if (errorRedirectThread != null) {
574 errorRedirectThread.stopRun();
575 errorRedirectThread = null;
576 }
577 } finally {
578 //if still running, kill it
579 killSolr();
580 }
581
582 logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
583 }
584 }
585
593 synchronized boolean isRunning() throws KeywordSearchModuleException {
594 try {
595
596 if (isPortAvailable(currentSolrServerPort)) {
597 return false;
598 }
599
600 // making a status request here instead of just doing solrServer.ping(), because
601 // that doesn't work when there are no cores
602 //TODO handle timeout in cases when some other type of server on that port
603 CoreAdminRequest.getStatus(null, localSolrServer);
604
605 logger.log(Level.INFO, "Solr server is running"); //NON-NLS
606 } catch (SolrServerException ex) {
607
608 Throwable cause = ex.getRootCause();
609
610 // TODO: check if SocketExceptions should actually happen (is
611 // probably caused by starting a connection as the server finishes
612 // shutting down)
613 if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
614 logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
615 return false;
616 } else {
617 throw new KeywordSearchModuleException(
618 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
619 }
620 } catch (SolrException ex) {
621 // Just log 404 errors for now...
622 logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
623 return false;
624 } catch (IOException ex) {
625 throw new KeywordSearchModuleException(
626 NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
627 }
628
629 return true;
630 }
631
632 /*
633 * ** Convenience methods for use while we only open one case at a time ***
634 */
644 void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
645 currentCoreLock.writeLock().lock();
646 try {
647 currentCore =
openCore(theCase, index);
648
649 try {
650 // execute a test query. if it fails, an exception will be thrown
652 } catch (NoOpenCoreException ex) {
653 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
654 }
655
656 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
657 } finally {
658 currentCoreLock.writeLock().unlock();
659 }
660 }
661
667 boolean coreIsOpen() {
668 currentCoreLock.readLock().lock();
669 try {
670 return (null != currentCore);
671 } finally {
672 currentCoreLock.readLock().unlock();
673 }
674 }
675
676 Index getIndexInfo() throws NoOpenCoreException {
677 currentCoreLock.readLock().lock();
678 try {
679 if (null == currentCore) {
680 throw new NoOpenCoreException();
681 }
682 return currentCore.getIndexInfo();
683 } finally {
684 currentCoreLock.readLock().unlock();
685 }
686 }
687
688 void closeCore() throws KeywordSearchModuleException {
689 currentCoreLock.writeLock().lock();
690 try {
691 if (null != currentCore) {
692 currentCore.close();
693 currentCore = null;
694 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
695 }
696 } finally {
697 currentCoreLock.writeLock().unlock();
698 }
699 }
700
701 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
702 currentCoreLock.readLock().lock();
703 try {
704 if (null == currentCore) {
705 throw new NoOpenCoreException();
706 }
707 currentCore.addDocument(doc);
708 } finally {
709 currentCoreLock.readLock().unlock();
710 }
711 }
712
722 @NbBundle.Messages({
723 "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
724 void deleteCore(String coreName) throws KeywordSearchServiceException {
725 /*
726 * Send a core unload request to the Solr server, with the parameters
727 * that request deleting the index and the instance directory
728 * (deleteInstanceDir removes everything related to the core, the index
729 * directory, the configuration files, etc.) set to true.
730 * NOTE: this method doesn't delete the actual Solr index directory. That is
731 * done as part of deleting case output directory.
732 */
733
734 // check whether the core we are deleting is the currently open core
735 currentCoreLock.readLock().lock();
736 try {
737 if (null != currentCore) {
738 if (currentCore.getName().equals(coreName)) {
739 // close current core first
740 closeCore();
741 }
742 }
743 } catch (KeywordSearchModuleException ex) {
744 throw new KeywordSearchServiceException(NbBundle.getMessage(Server.class, "Server.close.exception.msg"), ex);
745 } finally {
746 currentCoreLock.readLock().unlock();
747 }
748
749 try {
750 HttpSolrServer solrServer = new HttpSolrServer("http://" + UserPreferences.getIndexingServerHost() + ":" + UserPreferences.getIndexingServerPort() + "/solr"); //NON-NLS
751 connectToSolrServer(solrServer);
752 org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName,
true,
true, solrServer);
753 } catch (SolrServerException | IOException ex) {
754 throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
755 }
756 }
757
758
770 private Core
openCore(
Case theCase, Index index)
throws KeywordSearchModuleException {
771
772 try {
775 } else {
778 currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
779 }
780 connectToSolrServer(currentSolrServer);
781
782 } catch (SolrServerException | IOException ex) {
783 throw new KeywordSearchModuleException(NbBundle.getMessage(
Server.class,
"Server.connect.exception.msg"), ex);
784 }
785
786 try {
787 File dataDir =
new File(
new File(index.getIndexPath()).getParent());
// "data dir" is the parent of the index directory
789 dataDir.mkdirs();
790 }
791
792 if (!this.isRunning()) {
793 logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
794 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
795 }
796
797 String coreName = index.getIndexName();
799 /*
800 * The core either does not exist or it is not loaded. Make a
801 * request that will cause the core to be created if it does not
802 * exist or loaded if it already exists.
803 */
804
805 // In single user mode, if there is a core.properties file already,
806 // we've hit a solr bug. Compensate by deleting it.
809 if (corePropertiesFile.toFile().exists()) {
810 try {
811 corePropertiesFile.toFile().delete();
812 } catch (Exception ex) {
813 logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
814 }
815 }
816 }
817
818 CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
819 createCoreRequest.setDataDir(dataDir.getAbsolutePath());
820 createCoreRequest.setCoreName(coreName);
821 createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
822 createCoreRequest.setIsLoadOnStartup(false);
823 createCoreRequest.setIsTransient(true);
824 currentSolrServer.request(createCoreRequest);
825 }
826
828 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
829 }
830
831 return new Core(coreName, theCase.getCaseType(), index);
832
833 } catch (SolrServerException | SolrException | IOException ex) {
834 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
835 }
836 }
837
843 void commit() throws SolrServerException, NoOpenCoreException {
844 currentCoreLock.readLock().lock();
845 try {
846 if (null == currentCore) {
847 throw new NoOpenCoreException();
848 }
849 currentCore.commit();
850 } finally {
851 currentCoreLock.readLock().unlock();
852 }
853 }
854
855 NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
856 currentCoreLock.readLock().lock();
857 try {
858 if (null == currentCore) {
859 throw new NoOpenCoreException();
860 }
861 return currentCore.request(request);
862 } finally {
863 currentCoreLock.readLock().unlock();
864 }
865 }
866
878 currentCoreLock.readLock().lock();
879 try {
880 if (null == currentCore) {
881 throw new NoOpenCoreException();
882 }
883 try {
884 return currentCore.queryNumIndexedFiles();
885 } catch (SolrServerException | IOException ex) {
886 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
887 }
888 } finally {
889 currentCoreLock.readLock().unlock();
890 }
891 }
892
903 currentCoreLock.readLock().lock();
904 try {
905 if (null == currentCore) {
906 throw new NoOpenCoreException();
907 }
908 try {
909 return currentCore.queryNumIndexedChunks();
910 } catch (SolrServerException | IOException ex) {
911 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
912 }
913 } finally {
914 currentCoreLock.readLock().unlock();
915 }
916 }
917
928 currentCoreLock.readLock().lock();
929 try {
930 if (null == currentCore) {
931 throw new NoOpenCoreException();
932 }
933 try {
934 return currentCore.queryNumIndexedDocuments();
935 } catch (SolrServerException | IOException ex) {
936 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
937 }
938 } finally {
939 currentCoreLock.readLock().unlock();
940 }
941 }
942
953 public boolean queryIsIndexed(
long contentID)
throws KeywordSearchModuleException, NoOpenCoreException {
954 currentCoreLock.readLock().lock();
955 try {
956 if (null == currentCore) {
957 throw new NoOpenCoreException();
958 }
959 try {
960 return currentCore.queryIsIndexed(contentID);
961 } catch (SolrServerException | IOException ex) {
962 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
963 }
964
965 } finally {
966 currentCoreLock.readLock().unlock();
967 }
968 }
969
982 public int queryNumFileChunks(
long fileID)
throws KeywordSearchModuleException, NoOpenCoreException {
983 currentCoreLock.readLock().lock();
984 try {
985 if (null == currentCore) {
986 throw new NoOpenCoreException();
987 }
988 try {
989 return currentCore.queryNumFileChunks(fileID);
990 } catch (SolrServerException | IOException ex) {
991 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
992 }
993 } finally {
994 currentCoreLock.readLock().unlock();
995 }
996 }
997
1008 public QueryResponse
query(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1009 currentCoreLock.readLock().lock();
1010 try {
1011 if (null == currentCore) {
1012 throw new NoOpenCoreException();
1013 }
1014 try {
1015 return currentCore.query(sq);
1016 } catch (SolrServerException ex) {
1017 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1018 }
1019 } finally {
1020 currentCoreLock.readLock().unlock();
1021 }
1022 }
1023
1035 public QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1036 currentCoreLock.readLock().lock();
1037 try {
1038 if (null == currentCore) {
1039 throw new NoOpenCoreException();
1040 }
1041 try {
1042 return currentCore.query(sq, method);
1043 } catch (SolrServerException | IOException ex) {
1044 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1045 }
1046 } finally {
1047 currentCoreLock.readLock().unlock();
1048 }
1049 }
1050
1061 public TermsResponse
queryTerms(SolrQuery sq)
throws KeywordSearchModuleException, NoOpenCoreException {
1062 currentCoreLock.readLock().lock();
1063 try {
1064 if (null == currentCore) {
1065 throw new NoOpenCoreException();
1066 }
1067 try {
1068 return currentCore.queryTerms(sq);
1069 } catch (SolrServerException | IOException ex) {
1070 throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1071 }
1072 } finally {
1073 currentCoreLock.readLock().unlock();
1074 }
1075 }
1076
1087 currentCoreLock.readLock().lock();
1088 try {
1089 if (null == currentCore) {
1090 throw new NoOpenCoreException();
1091 }
1092 return currentCore.getSolrContent(content.getId(), 0);
1093 } finally {
1094 currentCoreLock.readLock().unlock();
1095 }
1096 }
1097
1111 currentCoreLock.readLock().lock();
1112 try {
1113 if (null == currentCore) {
1114 throw new NoOpenCoreException();
1115 }
1116 return currentCore.getSolrContent(content.getId(), chunkID);
1117 } finally {
1118 currentCoreLock.readLock().unlock();
1119 }
1120 }
1121
1132 currentCoreLock.readLock().lock();
1133 try {
1134 if (null == currentCore) {
1135 throw new NoOpenCoreException();
1136 }
1137 return currentCore.getSolrContent(objectID, 0);
1138 } finally {
1139 currentCoreLock.readLock().unlock();
1140 }
1141 }
1142
1153 public String
getSolrContent(
final long objectID,
final int chunkID)
throws NoOpenCoreException {
1154 currentCoreLock.readLock().lock();
1155 try {
1156 if (null == currentCore) {
1157 throw new NoOpenCoreException();
1158 }
1159 return currentCore.getSolrContent(objectID, chunkID);
1160 } finally {
1161 currentCoreLock.readLock().unlock();
1162 }
1163 }
1164
1176 }
1177
1186 void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1187 CoreAdminRequest.getStatus(null, solrServer);
1188 }
1189
1203 private boolean coreIsLoaded(String coreName)
throws SolrServerException, IOException {
1204 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1205 return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1206 }
1207
1220 CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1221 Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1222 if (null != dataDirPath) {
1223 File indexDir = Paths.get((String) dataDirPath,
"index").toFile();
//NON-NLS
1224 return indexDir.
exists();
1225 } else {
1226 return false;
1227 }
1228 }
1229
1230 class Core {
1231
1232 // handle to the core in Solr
1233 private final String name;
1234
1236
1237 private final Index textIndex;
1238
1239 // the server to access a core needs to be built from a URL with the
1240 // core in it, and is only good for core-specific operations
1241 private final HttpSolrServer solrCore;
1242
1243 private Core(String name,
CaseType caseType, Index index) {
1244 this.name = name;
1245 this.caseType = caseType;
1246 this.textIndex = index;
1247
1248 this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1249
1250 //TODO test these settings
1251 //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
1252 //solrCore.setConnectionTimeout(1000);
1253 solrCore.setDefaultMaxConnectionsPerHost(2);
1254 solrCore.setMaxTotalConnections(5);
1255 solrCore.setFollowRedirects(false); // defaults to false
1256 // allowCompression defaults to false.
1257 // Server side must support gzip or deflate for this to have any effect.
1258 solrCore.setAllowCompression(true);
1259 solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1260
1261 }
1262
1268 String getName() {
1269 return name;
1270 }
1271
1272 private Index getIndexInfo() {
1273 return this.textIndex;
1274 }
1275
1276 private QueryResponse
query(SolrQuery sq)
throws SolrServerException, IOException {
1277 return solrCore.query(sq);
1278 }
1279
1280 private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1281 try {
1282 return solrCore.request(request);
1283 } catch (IOException e) {
1284 logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1285 throw new SolrServerException(
1286 NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1287 }
1288
1289 }
1290
1291 private QueryResponse
query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1292 return solrCore.query(sq, method);
1293 }
1294
1295 private TermsResponse
queryTerms(SolrQuery sq)
throws SolrServerException, IOException {
1296 QueryResponse qres = solrCore.query(sq);
1297 return qres.getTermsResponse();
1298 }
1299
1300 private void commit() throws SolrServerException {
1301 try {
1302 //commit and block
1303 solrCore.commit(true, true);
1304 } catch (IOException e) {
1305 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1306 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1307 }
1308 }
1309
1310 void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1311 try {
1312 solrCore.add(doc);
1313 } catch (SolrServerException ex) {
1314 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1315 throw new KeywordSearchModuleException(
1316 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1317 } catch (IOException ex) {
1318 logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1319 throw new KeywordSearchModuleException(
1320 NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1321 }
1322 }
1323
1333 final SolrQuery q = new SolrQuery();
1334 q.setQuery("*:*");
1335 String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1336 if (chunkID != 0) {
1337 filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1338 }
1339 q.addFilterQuery(filterQuery);
1340 q.setFields(Schema.TEXT.toString());
1341 try {
1342 // Get the first result.
1343 SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1344
1345 if (!solrDocuments.isEmpty()) {
1346 SolrDocument solrDocument = solrDocuments.get(0);
1347 if (solrDocument != null) {
1348 Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1349 if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1350 {
1351 return fieldValues.toArray(new String[0])[0];
1352 } else // The indexed text for files has 2 values, the file name and the file content.
1353 // We return the file content value.
1354 {
1355 return fieldValues.toArray(new String[0])[1];
1356 }
1357 }
1358 }
1359 } catch (SolrServerException ex) {
1360 logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1361 return null;
1362 }
1363
1364 return null;
1365 }
1366
1367 synchronized void close() throws KeywordSearchModuleException {
1368 // We only unload cores for "single-user" cases.
1370 return;
1371 }
1372
1373 try {
1374 CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1375 } catch (SolrServerException ex) {
1376 throw new KeywordSearchModuleException(
1377 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1378 } catch (IOException ex) {
1379 throw new KeywordSearchModuleException(
1380 NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1381 }
1382 }
1383
1395 }
1396
1407 SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1408 q.setRows(0);
1409 int numChunks = (int)
query(q).getResults().getNumFound();
1410 return numChunks;
1411 }
1412
1424 SolrQuery q = new SolrQuery("*:*");
1425 q.setRows(0);
1426 return (
int)
query(q).getResults().getNumFound();
1427 }
1428
1438 private boolean queryIsIndexed(
long contentID)
throws SolrServerException, IOException {
1439 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1440 SolrQuery q = new SolrQuery("*:*");
1441 q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1442 //q.setFields(Server.Schema.ID.toString());
1443 q.setRows(0);
1444 return (
int)
query(q).getResults().getNumFound() != 0;
1445 }
1446
1458 private int queryNumFileChunks(
long contentID)
throws SolrServerException, IOException {
1459 String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1460 final SolrQuery q
1461 = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1462 q.setRows(0);
1463 return (
int)
query(q).getResults().getNumFound();
1464 }
1465 }
1466
1467 class ServerAction extends AbstractAction {
1468
1469 private static final long serialVersionUID = 1L;
1470
1471 @Override
1472 public void actionPerformed(ActionEvent e) {
1473 logger.log(Level.INFO, e.paramString().trim());
1474 }
1475 }
1476
1480 class SolrServerNoPortException extends SocketException {
1481
1482 private static final long serialVersionUID = 1L;
1483
1487 private final int port;
1488
1489 SolrServerNoPortException(int port) {
1490 super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1491 Server.PROPERTIES_CURRENT_SERVER_PORT));
1492 this.port = port;
1493 }
1494
1495 int getPortNumber() {
1496 return port;
1497 }
1498 }
1499 }
int queryNumIndexedFiles()
String getSolrContent(final long objectID)
final ReentrantReadWriteLock currentCoreLock
int queryNumIndexedChunks()
static final char ID_CHUNK_SEP
final ServerAction serverAction
static String getIndexingServerPort()
static final String CORE_PROPERTIES
boolean coreIsLoaded(String coreName)
final HttpSolrServer localSolrServer
static final Logger logger
void addServerActionListener(PropertyChangeListener l)
static final String HL_ANALYZE_CHARS_UNLIMITED
Process runSolrCommand(List< String > solrArguments)
String getSolrContent(final Content content)
static final long MAX_CONTENT_SIZE
boolean coreIndexFolderExists(String coreName)
HttpSolrServer currentSolrServer
int queryNumIndexedDocuments()
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final String CHUNK_ID_SEPARATOR
static final String CORE_EVT
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
InputStreamPrinterThread errorRedirectThread
QueryResponse query(SolrQuery sq)
static String getConfigSetting(String moduleName, String settingName)
int currentSolrServerPort
TermsResponse queryTerms(SolrQuery sq)
boolean queryIsIndexed(long contentID)
String getSolrContent(final Content content, int chunkID)
String getSolrContent(final long objectID, final int chunkID)
synchronized static Logger getLogger(String name)
Core openCore(Case theCase, Index index)
static final int MAX_SOLR_MEM_MB
static String getChunkIdString(long parentID, int childID)
static String getIndexingServerHost()
static boolean settingExists(String moduleName, String settingName)
int queryNumFileChunks(long fileID)
String toString(boolean preserveState)
static final boolean DEBUG
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)