并发写入多个文件

[我终于有blog了]

根据用户接口传递的数据组别,根据组别名字建立不同文件，实时将数据写入到不同文件。jsp页面查询数据传递是否结束，结束后下载文件。
1.http接口 servlet
package servlet;

import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;

import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import util.create_file;



/**
* Servlet implementation class encrypt_2md5
*/
@WebServlet("/encrypt_2md5")
public class encrypt_2md5 extends HttpServlet {
private static final long serialVersionUID = 1L;
//这个全局变量用于存放每一个session_id所对应的号码,不同的号码要写到不同的地方
ConcurrentHashMap<String,BlockingQueue<String>> session_map = new ConcurrentHashMap<String,BlockingQueue<String>>();

/**
* @see HttpServlet#HttpServlet()
*/
public encrypt_2md5() {
new Thread(new create_file(session_map)).start();
}

/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
// TODO Auto-generated method stub
response.getWriter().append("Served at: ").append(request.getContextPath());
}

/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
// TODO Auto-generated method stub
doGet(request, response);
}

@Override
public void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
String session = request.getParameter("session");
// JSONObject ob = JSONObject.fromObject(mobiles);
List<String> lis = new ArrayList<String>();
for (int i=0;i<5;i++) {
lis.add(i+"");
}
int i =0 ;
while (i<lis.size()){
String key = session;
String info = i+"";
//后台线程对这个map进行超时60s进行删除数据
if (session_map.containsKey(key)) {
try {
session_map.get(key).put(info);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}else {
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
try {
queue.put(info);
session_map.put(key,queue);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
i++;
}
PrintWriter write = response.getWriter();
write.append("1");
write.close();


}
}

2.处理queue的线程

package util;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;


public class create_file extends Thread {
//自增长的map,key是session_id,value是uid和其他值
ConcurrentHashMap<String,BlockingQueue<String>> map = null;
//正在跑的一些session,当上述的queue里面没有可以写的数据30s的时候,关闭文件
Set<String> session_set = ConcurrentHashMap.<String> newKeySet();
//用线程池来开启线程,方便知道一些事情
ExecutorService pool = Executors.newFixedThreadPool(10);
//这个可以并发读写,但是可能造成一定的延时性
List<Future<String>> resultList = new CopyOnWriteArrayList<Future<String>>();

public create_file(ConcurrentHashMap<String,BlockingQueue<String>> map) {
this.map = map;
new Thread(
new Runnable() {
@Override
public void run() {
while (true) {
//遍历任务的结果
Iterator<Future<String>> iter = resultList.iterator();
while (iter.hasNext()) {
try {
Future<String> fs = iter.next();
//这个会阻塞
String result = fs.get();
//当发现这个数据已经处理完,删除这个key，先要删除map里面的数据,然后再删除set里面的数据
map.remove(result);
//当发现这个没有数据之后删除掉这个key
session_set.remove(result);
//这个futurelist也要删除
resultList.remove(fs);
System.out.println("处理完session_id:"+result+"的数据");
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}

}


}
}

).start();

}

@Override
public void run() {
while (true) {
if(map != null&&map.size()>0) {
Set<String> keys = map.keySet();
Iterator<String> itor = keys.iterator();
while(itor.hasNext()) {
String key = itor.next();
//这个用于判断是否正在跑数据
if(session_set.contains(key)) {
continue;
}else {
System.out.println("开始处理session_id:"+key);
//没有的话就添加一个线程用于跑,处理Queue的数据
Future<String> b = pool.submit(new create_file_worker(key,map.get(key),map));
//锁住对象当增加数据的时候
resultList.add(b);
//增加正在跑的session_id
session_set.add(key);

}
}
}
}
}


}
3.写入文件的线程 （锁文件，用于判断是否处理完数据）
package util;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

/**
*
* @author DJBGG
* 在写入的时候对文件加锁，后续有进程访问文件遍历文件和下载文件
*/
public class create_file_worker implements Callable<String> {
final String key;
final BlockingQueue<String> queue;
final ConcurrentHashMap<String,BlockingQueue<String>> map;
private static String directory_path = "D:\\temp\\";

public create_file_worker(String key,BlockingQueue<String> queue,ConcurrentHashMap<String,BlockingQueue<String>> map) {
this.key = key;
this.queue = queue;
this.map = map;
}
@Override
public String call() throws Exception {
FileChannel channel = null;
FileLock lock = null;
RandomAccessFile raf = null;
try {
File file = new File(directory_path+key);
if (file.exists()) {

}else {
file = new File(directory_path+key);
file.createNewFile();
}
raf = new RandomAccessFile(directory_path+key,"rw");
//在文件末尾追加内容的处理
raf.seek(raf.length());
channel = raf.getChannel();
lock = channel.lock();//无参lock()为独占锁
// FileWriter write = new FileWriter(file,true);
int i =0;
while (true) {
try {
i = i + 1;
String info = queue.poll(30, TimeUnit.SECONDS);
if(info == null) {
throw new Exception();
}
// write.write(info+"\n");
//互斥操作
ByteBuffer sendBuffer=ByteBuffer.wrap((info+"\n").getBytes());
channel.write(sendBuffer);
}catch(Exception e) {
System.out.println("队列session:"+key+"应该没数据了");
//先关闭流再输出
// write.close();
if (lock != null) {
try {
lock.release();
lock = null;
} catch (IOException e1) {
e1.printStackTrace();
}
}

if (channel != null) {
try {
channel.close();
channel = null;
} catch (IOException e2) {
e2.printStackTrace();
}
}
if (raf != null) {
try {
raf.close();
raf = null;
} catch (IOException e1) {
e1.printStackTrace();
}
}
return key;
}

}
}catch(Exception e) {
System.out.println(e);
return key;
}finally {
if (lock != null) {
try {
lock.release();
lock = null;
} catch (IOException e3) {
e3.printStackTrace();
}
}

if (channel != null) {
try {
channel.close();
channel = null;
} catch (IOException e4) {
e4.printStackTrace();
}
if (raf != null) {
try {
raf.close();
raf = null;
} catch (IOException e1) {
e1.printStackTrace();
}
}
}
}

}

}