package searchfileexample;
/**
* 讀取Excel文件
*/
import java.io.*;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import java.util.Date;
import org.apache.poi.hssf.usermodel.HSSFRow;
public class ExcelReader {
// 創(chuàng)建文件輸入流
private BufferedReader reader = null;
// 文件類型
private String filetype;
// 文件二進制輸入流
private InputStream is = null;
// 當(dāng)前的Sheet
private int currSheet;
// 當(dāng)前位置
private int currPosition;
// Sheet數(shù)量
private int numOfSheets;
// HSSFWorkbook
HSSFWorkbook workbook = null;
// 設(shè)置Cell之間以空格分割
private static String EXCEL_LINE_DELIMITER = " ";
// 設(shè)置最大列數(shù)
private static int MAX_EXCEL_COLUMNS = 64;
public int rows = 0;
public int getRows() {
return rows;
}
// 構(gòu)造函數(shù)創(chuàng)建一個ExcelReader
public ExcelReader(String inputfile) throws IOException, Exception {
// 判斷參數(shù)是否為空或沒有意義
if (inputfile == null || inputfile.trim().equals("")) {
throw new IOException("no input file specified");
}
// 取得文件名的后綴名賦值給filetype
this.filetype = inputfile.substring(inputfile.lastIndexOf(".") + 1);
// 設(shè)置開始行為0
currPosition = 0;
// 設(shè)置當(dāng)前位置為0
currSheet = 0;
// 創(chuàng)建文件輸入流
is = new FileInputStream(inputfile);
// 判斷文件格式
if (filetype.equalsIgnoreCase("txt")) {
// 如果是txt則直接創(chuàng)建BufferedReader讀取
reader = new BufferedReader(new InputStreamReader(is));
}
else if (filetype.equalsIgnoreCase("xls")) {
// 如果是Excel文件則創(chuàng)建HSSFWorkbook讀取
workbook = new HSSFWorkbook(is);
// 設(shè)置Sheet數(shù)
numOfSheets = workbook.getNumberOfSheets();
}
else {
throw new Exception("File Type Not Supported");
}
}
// 函數(shù)readLine讀取文件的一行
public String readLine() throws IOException {
// 如果是txt文件則通過reader讀取
if (filetype.equalsIgnoreCase("txt")) {
String str = reader.readLine();
// 空行則略去，直接讀取下一行
while (str.trim().equals("")) {
str = reader.readLine();
}
return str;
}
// 如果是XLS文件則通過POI提供的API讀取文件
else if (filetype.equalsIgnoreCase("xls")) {
// 根據(jù)currSheet值獲得當(dāng)前的sheet
HSSFSheet sheet = workbook.getSheetAt(currSheet);
rows = sheet.getLastRowNum();
// 判斷當(dāng)前行是否到但前Sheet的結(jié)尾
if (currPosition > sheet.getLastRowNum()) {
// 當(dāng)前行位置清零
currPosition = 0;
// 判斷是否還有Sheet
while (currSheet != numOfSheets - 1) {
// 得到下一張Sheet
sheet = workbook.getSheetAt(currSheet + 1);
// 當(dāng)前行數(shù)是否已經(jīng)到達(dá)文件末尾
if (currPosition == sheet.getLastRowNum()) {
// 當(dāng)前Sheet指向下一張Sheet
currSheet++;
continue;
}
else {
// 獲取當(dāng)前行數(shù)
int row = currPosition;
currPosition++;
// 讀取當(dāng)前行數(shù)據(jù)
return getLine(sheet, row);
}
}
return null;
}
// 獲取當(dāng)前行數(shù)
int row = currPosition;
currPosition++;
// 讀取當(dāng)前行數(shù)據(jù)
return getLine(sheet, row);
}
return null;
}
// 函數(shù)getLine返回Sheet的一行數(shù)據(jù)
private String getLine(HSSFSheet sheet, int row) {
// 根據(jù)行數(shù)取得Sheet的一行
HSSFRow rowline = sheet.getRow(row);
// 創(chuàng)建字符創(chuàng)緩沖區(qū)
StringBuffer buffer = new StringBuffer();
// 獲取當(dāng)前行的列數(shù)
int filledColumns = rowline.getLastCellNum();
HSSFCell cell = null;
// 循環(huán)遍歷所有列
for (int i = 0; i < filledColumns; i++) {
// 取得當(dāng)前Cell
cell = rowline.getCell( (short) i);
String cellvalue = null;
if (cell != null) {
// 判斷當(dāng)前Cell的Type
switch (cell.getCellType()) {
// 如果當(dāng)前Cell的Type為NUMERIC
case HSSFCell.CELL_TYPE_NUMERIC: {
// 判斷當(dāng)前的cell是否為Date
if (HSSFDateUtil.isCellDateFormatted(cell)) {
// 如果是Date類型則，取得該Cell的Date值
Date date = cell.getDateCellValue();
// 把Date轉(zhuǎn)換成本地格式的字符串
cellvalue = cell.getDateCellValue().toLocaleString();
}
// 如果是純數(shù)字
else {
// 取得當(dāng)前Cell的數(shù)值
Integer num = new Integer( (int) cell
.getNumericCellValue());
cellvalue = String.valueOf(num);
}
break;
}
// 如果當(dāng)前Cell的Type為STRIN
case HSSFCell.CELL_TYPE_STRING:
// 取得當(dāng)前的Cell字符串
cellvalue = cell.getStringCellValue().replaceAll("'", "''");
break;
// 默認(rèn)的Cell值
default:
cellvalue = " ";
}
}
else {
cellvalue = "";
}
// 在每個字段之間插入分割符
buffer.append(cellvalue).append(EXCEL_LINE_DELIMITER);
}
// 以字符串返回該行的數(shù)據(jù)
return buffer.toString();
}
// close函數(shù)執(zhí)行流的關(guān)閉操作
public void close() {
// 如果is不為空，則關(guān)閉InputSteam文件輸入流
if (is != null) {
try {
is.close();
}
catch (IOException e) {
is = null;
}
}
// 如果reader不為空則關(guān)閉BufferedReader文件輸入流
if (reader != null) {
try {
reader.close();
}
catch (IOException e) {
reader = null;
}
}
}
public static void main(String[] args) {
try {
ExcelReader er = new ExcelReader("d:\\xp.xls");
String line = er.readLine();
while (line != null) {
System.out.println(line);
line = er.readLine();
}
er.close();
}
catch (Exception e) {
e.printStackTrace();
}
}
}
package searchfileexample;
import javax.servlet.*;
import javax.servlet.http.*;
import java.io.*;
import java.util.*;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Date;
import org.apache.lucene.demo.FileDocument;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import java.io.FileReader;
import org.apache.lucene.index.*;
import java.text.DateFormat;
import org.apache.poi.hdf.extractor.WordDocument;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.PrintWriter;
import java.io.FileInputStream;
import java.io.*;
import org.textmining.text.extraction.WordExtractor;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
/**
* 給某個目錄下的所有文件生成索引
* <p>Title: </p>
* <p>Description: </p>
* <p>Copyright: Copyright (c) 2007</p>
* <p>Company: </p>
* @author not attributable
* @version 1.0
* 根據(jù)文件的不同，可以把索引文件創(chuàng)建到不同的文件夾下去，這樣可以分類保存索引信息。
*/
public class IndexFilesServlet
extends HttpServlet {
static final File INDEX_DIR = new File("index");
//Initialize global variables
public void init() throws ServletException {
}
//Process the HTTP Get request
public void service(HttpServletRequest request, HttpServletResponse response) throws
ServletException, IOException {
final File docDir = new File("a"); //需要生成索引的文件的文件夾
if (!docDir.exists() || !docDir.canRead()) {
System.out.println("Document directory '" + docDir.getAbsolutePath() +
"' does not exist or is not readable, please check the path");
System.exit(1);
}
Date start = new Date();
try {
IndexWriter writer = new IndexWriter(INDEX_DIR, new StandardAnalyzer(), true); //true-覆蓋原有的索引 false-不覆蓋原有的索引
System.out.println("Indexing to directory '" + INDEX_DIR + "'...");
indexDocs(writer, docDir);
System.out.println("Optimizing...");
writer.optimize();
writer.close();
Date end = new Date();
System.out.println(end.getTime() - start.getTime() +
" total milliseconds");
}
catch (IOException e) {
System.out.println(" caught a " + e.getClass() +
"\n with message: " + e.getMessage());
}
}
//Clean up resources
public void destroy() {
}
public void indexDocs(IndexWriter writer, File file) throws IOException {
// do not try to index files that cannot be read
int index = 0;
String filehouzui = "";
index = file.getName().indexOf(".");
//strFileName = strFileName.substring(0, index) +DateUtil.getCurrDateTime() + "." + strFileName.substring(index + 1);
filehouzui = file.getName().substring(index + 1);
if (file.canRead()) {
if (file.isDirectory()) {
String[] files = file.list();
// an IO error could occur
if (files != null) {
for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i]));
}
}
}
else {
System.out.println("adding " + file);
try {
if (filehouzui.equals("doc")) {
writer.addDocument(getWordDocument(file, new FileInputStream(file)));
}
else if (filehouzui.equals("txt")) {
writer.addDocument(getTxtDocument(file, new FileInputStream(file)));
}
else if (filehouzui.equals("xls")) {
writer.addDocument(getExcelDocument(file, new FileInputStream(file)));
}
//writer.addDocument(parseFile(file));
//writer.addDocument(FileDocument.Document(file));//path 存放文件的相對路徑
}
// at least on windows, some temporary files raise this exception with an "access denied" message
// checking if the file can be read doesn't help
catch (Exception fnfe) {
;
}
}
}
}
/**
*@paramfile
*
*把File變成Document
*/
public Document parseFile(File file) throws Exception {
Document doc = new Document();
doc.add(new Field("path", file.getAbsolutePath(), Field.Store.YES,
Field.Index.UN_TOKENIZED)); //取文件的絕對路徑
try {
doc.add(new Field("contents", new FileReader(file))); //索引文件內(nèi)容
doc.add(new Field("title", file.getName(), Field.Store.YES,
Field.Index.UN_TOKENIZED));
//索引最后修改時間
doc.add(new Field("modified",
String.valueOf(DateFormat.
getDateTimeInstance().format(new
Date(file.lastModified()))), Field.Store.YES,
Field.Index.UN_TOKENIZED));
//doc.removeField("title");
}
catch (Exception e) {
e.printStackTrace();
}
return doc;
}
/**
*@paramfile
*
*使用POI讀取word文檔
* 不太好用，讀取word文檔不全
*/
public Document getDocument(File file, FileInputStream is) throws Exception {
String bodyText = null;
try {
WordDocument wd = new WordDocument(is);
StringWriter docTextWriter = new StringWriter();
wd.writeAllText(new PrintWriter(docTextWriter));
bodyText = docTextWriter.toString();
docTextWriter.close();
// bodyText = new WordExtractor().extractText(is);
System.out.println("word content====" + bodyText);
}
catch (Exception e) {
;
}
if ( (bodyText != null)) {
Document doc = new Document();
doc.add(new Field("path", file.getAbsolutePath(), Field.Store.YES,
Field.Index.UN_TOKENIZED)); //取文件的絕對路徑
doc.add(new Field("contents", bodyText, Field.Store.YES,
Field.Index.TOKENIZED));
return doc;
}
return null;
}
//Document doc = getDocument(new FileInputStream(new File(file)));
/**
*@paramfile
*
*使用tm-extractors-0.4.jar讀取word文檔
* 好用
*/
public Document getWordDocument(File file, FileInputStream is) throws
Exception {
String bodyText = null;
try {
WordExtractor extractor = new WordExtractor();
System.out.println("word文檔");
bodyText = extractor.extractText(is);
if ( (bodyText != null)) {
Document doc = new Document();
doc.add(new Field("path", file.getAbsolutePath(), Field.Store.YES,
Field.Index.UN_TOKENIZED)); //取文件的絕對路徑
doc.add(new Field("contents", bodyText, Field.Store.YES,
Field.Index.TOKENIZED));
System.out.println("word content====" + bodyText);
return doc;
}
}
catch (Exception e) {
;
}
return null;
}
/**
*@paramfile
*
*讀取TXT文檔
*/
public Document getTxtDocument(File file, FileInputStream is) throws
Exception {
try {
Reader textReader = new FileReader(file);
Document doc = new Document();
doc.add(new Field("path", file.getAbsolutePath(), Field.Store.YES,
Field.Index.UN_TOKENIZED)); //取文件的絕對路徑
doc.add(new Field("contents", textReader));
return doc;
}
catch (Exception e) {
;
}
return null;
}
/**
* 使用POI讀取Excel文件
* @param file File
* @param is FileInputStream
* @throws Exception
* @return Document
*/
public Document getExcelDocument(File file, FileInputStream is) throws
Exception {
String bodyText = "";
try {
System.out.println("讀取excel文件");
ExcelReader er = new ExcelReader(file.getAbsolutePath());
bodyText = er.readLine();
int rows = 0;
rows = er.getRows();
for (int i = 0; i < rows; i++) {
bodyText = bodyText + er.readLine();
System.out.println("bodyText===" + bodyText);
}
Document doc = new Document();
doc.add(new Field("path", file.getAbsolutePath(), Field.Store.YES,
Field.Index.UN_TOKENIZED)); //取文件的絕對路徑
doc.add(new Field("contents", bodyText, Field.Store.YES,
Field.Index.TOKENIZED));
System.out.println("word content====" + bodyText);
return doc;
}
catch (Exception e) {
System.out.println(e);
}
return null;
}
}
package searchfileexample;
import javax.servlet.*;
import javax.servlet.http.*;
import java.io.*;
import java.util.*;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searcher;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;
import org.apache.lucene.queryParser.*;
public class SearchFileServlet
extends HttpServlet {
private static final String CONTENT_TYPE = "text/html; charset=GBK";
//Initialize global variables
public void init() throws ServletException {
}
/** Use the norms from one field for all fields. Norms are read into memory,
* using a byte of memory per document per searched field. This can cause
* search of large collections with a large number of fields to run out of
* memory. If all of the fields contain only a single token, then the norms
* are all identical, then single norm vector may be shared. */
private static class OneNormsReader
extends FilterIndexReader {
private String field;
public OneNormsReader(IndexReader in, String field) {
super(in);
this.field = field;
}
public byte[] norms(String field) throws IOException {
return in.norms(this.field);
}
}
//Process the HTTP Get request
public void service(HttpServletRequest request, HttpServletResponse response) throws
ServletException, IOException {
response.setContentType(CONTENT_TYPE);
PrintWriter out = response.getWriter();
String[] args = {
"a", "b"};
String usage =
"Usage: java org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-raw] [-norms field]";
if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
System.out.println(usage);
System.exit(0);
}
String index = "index"; //該值是用來存放生成的索引文件的文件夾的名稱，不能改動
String field = "contents"; //不能修改 field 的值
String queries = null; //是用來存放需要檢索的關(guān)鍵字的一個文件。
queries = "D:/lfy_programe/全文檢索/SearchFileExample/aa.txt";
System.out.println("-----------------------" + request.getContextPath());
int repeat = 1;
boolean raw = false;
String normsField = null;
for (int i = 0; i < args.length; i++) {
if ("-index".equals(args[i])) {
index = args[i + 1];
i++;
}
else if ("-field".equals(args[i])) {
field = args[i + 1];
i++;
}
else if ("-queries".equals(args[i])) {
queries = args[i + 1];
i++;
}
else if ("-repeat".equals(args[i])) {
repeat = Integer.parseInt(args[i + 1]);
i++;
}
else if ("-raw".equals(args[i])) {
raw = true;
}
else if ("-norms".equals(args[i])) {
normsField = args[i + 1];
i++;
}
}
IndexReader reader = IndexReader.open(index);
if (normsField != null) {
reader = new OneNormsReader(reader, normsField);
}
Searcher searcher = new IndexSearcher(reader); //用來打開索引文件
Analyzer analyzer = new StandardAnalyzer(); //分析器
//Analyzer analyzer = new StandardAnalyzer();
BufferedReader in = null;
if (queries != null) {
in = new BufferedReader(new FileReader(queries));
}
else {
in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
}
QueryParser parser = new QueryParser(field, analyzer);
out.println("<html>");
out.println("<head><title>SearchFileServlet</title></head>");
out.println("<body bgcolor=\"#ffffff\">");
while (true) {
if (queries == null) { // prompt the user
System.out.println("Enter query: ");
}
String line = in.readLine(); //組成查詢關(guān)鍵字字符串
System.out.println("查詢字符串===" + line);
if (line == null || line.length() == -1) {
break;
}
line = line.trim();
if (line.length() == 0) {
break;
}
Query query = null;
try {
query = parser.parse(line);
}
catch (ParseException ex) {
}
System.out.println("Searching for: " + query.toString(field)); //每個關(guān)鍵字
Hits hits = searcher.search(query);
if (repeat > 0) { // repeat & time as benchmark
Date start = new Date();
for (int i = 0; i < repeat; i++) {
hits = searcher.search(query);
}
Date end = new Date();
System.out.println("Time: " + (end.getTime() - start.getTime()) + "ms");
}
out.println("<p>查詢到：" + hits.length() + "個含有[" +
query.toString(field) + "]的文檔</p>");
System.out.println("查詢到：" + hits.length() + " 個含有 [" +
query.toString(field) + "]的文檔");
final int HITS_PER_PAGE = 10; //查詢返回的最大記錄數(shù)
int currentNum = 5; //當(dāng)前記錄數(shù)
for (int start = 0; start < hits.length(); start += HITS_PER_PAGE) {
//start = start + currentNum;
int end = Math.min(hits.length(), start + HITS_PER_PAGE);
for (int i = start; i < end; i++) {
//if (raw) { // output raw format
System.out.println("doc=" + hits.id(i) + " score=" + hits.score(i)); //score是接近度的意思
//continue;
//}
Document doc = hits.doc(i);
String path = doc.get("path");
if (path != null) {
System.out.println( (i + 1) + ". " + path);
out.println("<p>" + (i + 1) + ". " + path + "</p>");
String title = doc.get("title");
System.out.println(" modified: " + doc.get("modified"));
if (title != null) {
System.out.println(" Title: " + doc.get("title"));
}
}
else {
System.out.println( (i + 1) + ". " + "No path for this document");
}
}
if (queries != null) { // non-interactive
break;
}
if (hits.length() > end) {
System.out.println("more (y/n) ? ");
line = in.readLine();
if (line.length() == 0 || line.charAt(0) == 'n') {
break;
}
}
}
}
reader.close();
out.println("</body></html>");
}
//Clean up resources
public void destroy() {
}
}