lucene2.0+heritrix½CÞZ¾‹è¡¥å……

Mon, 09 Feb 2009 07:44:00 GMT

searchçš„Extractorä»£ç å¦‚ä¸‹åQŒï¼ˆåˆ«å’Œä¹¦ä¸Šå®žä¾‹ç›¸åŒåQ‰ä¾›å¤§å®¶å‚è€ƒï¼š(x¨¬)é™„äšgé‡Œæœ‰å®Œæ•´ä»£ç

package com.luceneheritrixbook.extractor.younet;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Date;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.HasChildFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;

import com.luceneheritrixbook.extractor.Extractor;
import com.luceneheritrixbook.util.StringUtils;

/**
*

* @author cnyqiao@hotmail.com
* @date   Feb 6, 2009
*/

public class ExtractYounetMoblie extends Extractor {

    @Override
    public void extract() {
        BufferedWriter bw = null;
        NodeFilter title_filter = new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "mo_tit"));
        NodeFilter attribute_filter = new AndFilter(new TagNameFilter("p"), new HasChildFilter(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("class", "gn_sp1 blue1"))));
        NodeFilter img_filter = new AndFilter(new TagNameFilter("span"), new HasChildFilter(new TagNameFilter("img")));

        //æå–æ ‡é¢˜?sh¨´)¿¡æ?/span>
        try {
            //Parseræ ÒŽ(gu¨©)®˜q‡æ×o(h¨´)å™¨è¿”å›žæ‰€æœ‰æ»¡‘Œ™¿‡æ»¤æ¡ä»¶çš„èŠ‚ç‚¹
            // ˜qä»£é€æ¸æŸ¥æ‰¾
            NodeList nodeList=this.getParser().parse(title_filter);
            NodeIterator it = nodeList.elements();
            StringBuffer title = new StringBuffer();
            while (it.hasMoreNodes()) {
                Node node = (Node) it.nextNode();
                String[] names = node.toPlainTextString().split(" ");
                for(int i = 0; i < names.length; i++)
                    title.append(names[i]).append("-");
                title.append(new Date().getTime());
                //åˆ›å¾è¦ç”Ÿæˆçš„æ–‡äšg
                bw = new BufferedWriter(new FileWriter(new File(this.getOutputPath() + title + ".txt")));
                //èŽ·å–å½“å‰æå–™åëŠš„å®Œæ•´URLåœ°å€
                int startPos = this.getInuputFilePath().indexOf("mirror") + 6;
                String url_seg = this.getInuputFilePath().substring(startPos);
                url_seg = url_seg.replaceAll("\\\\", "/");
                String url = "http:/" + url_seg;
                //å†™å…¥å½“å‰æå–™åëŠš„å®Œæ•´URLåœ°å€
                bw.write(url + NEWLINE);
                bw.write(names[0] + NEWLINE);
                bw.write(names[1] + NEWLINE);

            }
            // é‡ç½®Parser
            this.getParser().reset();
            Parser attNameParser = null;
            Parser attValueParser = null;
            //Parser parser=new Parser("http://www.sina.com.cn");
            NodeFilter attributeName_filter = new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("class", "gn_sp1 blue1"));
            NodeFilter attributeValue_filter = new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("class", "gn_sp2"));
            String attName = "";
            String attValue = "";
            // ˜qä»£é€æ¸æŸ¥æ‰¾
            nodeList=this.getParser().parse(attribute_filter);
            it = nodeList.elements();
            while (it.hasMoreNodes()) {
                Node node = (Node) it.nextNode();
                attNameParser = new Parser();
                attNameParser.setEncoding("GB2312");
                attNameParser.setInputHTML(node.toHtml());
                NodeList attNameNodeList = attNameParser.parse(attributeName_filter);
                attName = attNameNodeList.elements().nextNode().toPlainTextString();

                attValueParser = new Parser();
                attValueParser.setEncoding("GB2312");
                attValueParser.setInputHTML(node.toHtml());
                NodeList attValueNodeList = attValueParser.parse(attributeValue_filter);
                attValue = attValueNodeList.elements().nextNode().toPlainTextString();
                bw.write(attName.trim() + attValue.trim());
                bw.newLine();
            }
            // é‡ç½®Parser
            this.getParser().reset();
            String imgUrl = "";
            String fileType ="";
            // ˜qä»£é€æ¸æŸ¥æ‰¾
            nodeList=this.getParser().parse(img_filter);
            it = nodeList.elements();
            while (it.hasMoreNodes()) {
                Node node = (Node) it.nextNode();

                ImageTag imgNode = (ImageTag)node.getChildren().elements().nextNode();
                imgUrl = imgNode.getAttribute("src");
                fileType = imgUrl.trim().substring(imgUrl
                        .lastIndexOf(".") + 1);
                //ç”Ÿæˆæ–°çš„å›„¡‰‡çš„æ–‡ä»¶å
                String new_iamge_file = StringUtils.encodePassword(imgUrl, HASH_ALGORITHM) + "." + fileType;
                //imgUrl = new HtmlPaserFilterTest().replace(new_iamge_file, "+", " ");
                //åˆ©ç”¨miorrç›®å½•ä¸‹çš„å›„¡‰‡ç”Ÿæˆçš„æ–°çš„å›¾ç‰?/span>
                this.copyImage(imgUrl, new_iamge_file);
                bw.write(SEPARATOR + NEWLINE);
                bw.write(new_iamge_file + NEWLINE);
            }


        } catch(Exception e) {
            e.printStackTrace();
        } finally {
            try{
                if (bw != null)
                    bw.close();
            }catch(IOException e){
                e.printStackTrace();
            }
        }

    }
}

˜qè¡Œä¹¦ä¸Šçš„heritrixå®žä¾‹åQŒåÆˆæŒ‰ä¹¦ä¸Šçš„é»˜è®¤è®„¡½®˜q›è¡ŒæŠ“å–å¦‚ä¸‹åQµï¼²åQ©ï¼š(x¨¬)åQˆè¯·è‡ªå·±åˆ†æžæ•´ç†åQ?br />

http://mobile.younet.com/files/list_1.html
http://mobile.younet.com/files/list_2.html
http://mobile.younet.com/files/list_3.html

ä¸€å‡?/a> 2009-02-09 15:44 å‘è¡¨è¯„è®º

]]>

欧美劲爆第一页,日韩成人影音,久久最新资源网

lucene2.0+heritrix½CÞZ¾‹è¡¥å……