IT神童

Java中用內(nèi)存映射處理大文件

在處理大文件時(shí)，如果利用普通的FileInputStream 或者FileOutputStream 抑或RandomAccessFile 來(lái)進(jìn)行頻繁的讀寫操作，都將導(dǎo)致進(jìn)程因頻繁讀寫外存而降低速度.如下為一個(gè)對(duì)比實(shí)驗(yàn)。

1. package test;  
2.  
3. import java.io.BufferedInputStream;  
4. import java.io.FileInputStream;  
5. import java.io.FileNotFoundException;  
6. import java.io.IOException;  
7. import java.io.RandomAccessFile;  
8. import java.nio.MappedByteBuffer;  
9. import java.nio.channels.FileChannel;  
10.  
11. public class Test {  
12.  
13.       
14.     public static void main(String[] args) {  
15.         try {  
16.             FileInputStream fis=new FileInputStream("/home/tobacco/test/res.txt");  
17.             int sum=0;  
18.             int n;  
19.             long t1=System.currentTimeMillis();  
20.             try {  
21.                 while((n=fis.read())>=0){  
22.                     sum+=n;  
23.                 }  
24.             } catch (IOException e) {  
25.                 // TODO Auto-generated catch block  
26.                 e.printStackTrace();  
27.             }  
28.             long t=System.currentTimeMillis()-t1;  
29.             System.out.println("sum:"+sum+"  time:"+t);  
30.         } catch (FileNotFoundException e) {  
31.             // TODO Auto-generated catch block  
32.             e.printStackTrace();  
33.         }  
34.           
35.         try {  
36.             FileInputStream fis=new FileInputStream("/home/tobacco/test/res.txt");  
37.             BufferedInputStream bis=new BufferedInputStream(fis);  
38.             int sum=0;  
39.             int n;  
40.             long t1=System.currentTimeMillis();  
41.             try {  
42.                 while((n=bis.read())>=0){  
43.                     sum+=n;  
44.                 }  
45.             } catch (IOException e) {  
46.                 // TODO Auto-generated catch block  
47.                 e.printStackTrace();  
48.             }  
49.             long t=System.currentTimeMillis()-t1;  
50.             System.out.println("sum:"+sum+"  time:"+t);  
51.         } catch (FileNotFoundException e) {  
52.             // TODO Auto-generated catch block  
53.             e.printStackTrace();  
54.         }  
55.           
56.         MappedByteBuffer buffer=null;  
57.         try {  
58.             buffer=new RandomAccessFile("/home/tobacco/test/res.txt","rw").getChannel().map(FileChannel.MapMode.READ_WRITE, 0, 1253244);  
59.             int sum=0;  
60.             int n;  
61.             long t1=System.currentTimeMillis();  
62.             for(int i=0;i<1253244;i++){  
63.                 n=0x000000ff&buffer.get(i);  
64.                 sum+=n;  
65.             }  
66.             long t=System.currentTimeMillis()-t1;  
67.             System.out.println("sum:"+sum+"  time:"+t);  
68.         } catch (FileNotFoundException e) {  
69.             // TODO Auto-generated catch block  
70.             e.printStackTrace();  
71.         } catch (IOException e) {  
72.             // TODO Auto-generated catch block  
73.             e.printStackTrace();  
74.         }  
75.  
76.     }  
77.  
78. }  

java基礎(chǔ)教程測(cè)試文件為一個(gè)大小為1253244字節(jié)的文件。測(cè)試結(jié)果：

sum:220152087 time:1464  
sum:220152087 time:72  
sum:220152087 time:25 

說(shuō)明讀數(shù)據(jù)無(wú)誤。刪去其中的數(shù)據(jù)處理部分。

package test;  
 
import java.io.BufferedInputStream;  
import java.io.FileInputStream;  
import java.io.FileNotFoundException;  
import java.io.IOException;  
import java.io.RandomAccessFile;  
import java.nio.MappedByteBuffer;  
import java.nio.channels.FileChannel;  
 
public class Test {  
 
      
    public static void main(String[] args) {  
        try {  
            FileInputStream fis=new FileInputStream("/home/tobacco/test/res.txt");  
            int sum=0;  
            int n;  
            long t1=System.currentTimeMillis();  
            try {  
                while((n=fis.read())>=0){  
                    //sum+=n;  
                }  
            } catch (IOException e) {  
                // TODO Auto-generated catch block  
                e.printStackTrace();  
            }  
            long t=System.currentTimeMillis()-t1;  
            System.out.println("sum:"+sum+"  time:"+t);  
        } catch (FileNotFoundException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        }  
          
        try {  
            FileInputStream fis=new FileInputStream("/home/tobacco/test/res.txt");  
            BufferedInputStream bis=new BufferedInputStream(fis);  
            int sum=0;  
            int n;  
            long t1=System.currentTimeMillis();  
            try {  
                while((n=bis.read())>=0){  
                    //sum+=n;  
                }  
            } catch (IOException e) {  
                // TODO Auto-generated catch block  
                e.printStackTrace();  
            }  
            long t=System.currentTimeMillis()-t1;  
            System.out.println("sum:"+sum+"  time:"+t);  
        } catch (FileNotFoundException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        }  
          
        MappedByteBuffer buffer=null;  
        try {  
            buffer=new RandomAccessFile("/home/tobacco/test/res.txt","rw").getChannel().map(FileChannel.MapMode.READ_WRITE, 0, 1253244);  
            int sum=0;  
            int n;  
            long t1=System.currentTimeMillis();  
            for(int i=0;i<1253244;i++){  
                //n=0x000000ff&buffer.get(i);  
                //sum+=n;  
            }  
            long t=System.currentTimeMillis()-t1;  
            System.out.println("sum:"+sum+"  time:"+t);  
        } catch (FileNotFoundException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        } catch (IOException e) {  
            // TODO Auto-generated catch block  
            e.printStackTrace();  
        }  
 
    }  
 
} 

測(cè)試結(jié)果：

sum:0 time:1458  
sum:0 time:67  
sum:0 time:8 

由此可見(jiàn)，將文件部分或者全部映射到內(nèi)存后進(jìn)行讀寫，速度將提高很多。

這是因?yàn)閮?nèi)存映射文件首先將外存上的文件映射到內(nèi)存中的一塊連續(xù)區(qū)域，被當(dāng)成一個(gè)字節(jié)數(shù)組進(jìn)行處理，讀寫操作直接對(duì)內(nèi)存進(jìn)行操作，而后再將內(nèi)存區(qū)域重新映射到外存文件，這就節(jié)省了中間頻繁的對(duì)外存進(jìn)行讀寫的時(shí)間，大大降低了讀寫時(shí)間。

posted on 2012-08-23 14:37 IT神童 閱讀(217) 評(píng)論(0)  編輯  收藏