62,621
社区成员
发帖
与我相关
我的任务
分享
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(conf.getInt(DEFAULT_THREAD_POOL_SIZE, 64));
executor.setRejectedExecutionHandler(retryHandler);
executor.execute(new ComputeRunnable(XXX));
executor.shutdown();
try {
executor.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new IOException(e);
}
package concurrencyTest;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class ExecutorReuse {
/**
* @param args
*/
public static void main(String[] args) {
try {
long start = System.currentTimeMillis() ;
RejectedExecutionHandler retryHandler = new RetryRejectedExecutionHandler();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(64);
executor.setRejectedExecutionHandler(retryHandler);
int i = 0;
while (i<10000) {
executor.execute(new ComputeRunnable(i));
i++ ;
}
executor.shutdown();
executor.awaitTermination(60, TimeUnit.SECONDS);
i=0 ;
while (i<100) {
executor.execute(new ComputeRunnable2(i));
i++ ;
}
executor.shutdown();
executor.awaitTermination(60, TimeUnit.SECONDS);
System.out.println("Last " +( System.currentTimeMillis() - start ) +" ms");
} catch (Exception e) {
e.printStackTrace() ;
}
}
}
class ComputeRunnable implements Runnable {
int id = 0 ;
public ComputeRunnable(int i ) {
id = i ;
}
@Override
public void run() {
id =id *9 ;
// System.out.println("ComputeRunnable1 started!");
}
}
class ComputeRunnable2 implements Runnable {
int id = 0 ;
public ComputeRunnable2(int i ) {
id = i ;
}
@Override
public void run() {
id =id *4 ;
System.out.println("ComputeRunnable2 started!");
}
}
package concurrencyTest;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class ExecutorReuse {
/**
* @param args
*/
public static void main(String[] args) {
try {
long start = System.currentTimeMillis() ;
RejectedExecutionHandler retryHandler = new RetryRejectedExecutionHandler();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(64);
executor.setRejectedExecutionHandler(retryHandler);
int i = 0;
while (i<10000) {
executor.execute(new ComputeRunnable(i));
i++ ;
}
executor.shutdown();
executor.awaitTermination(60, TimeUnit.SECONDS);
System.out.println("First 1 Step Last " +( System.currentTimeMillis() - start ) +" ms");
start = System.currentTimeMillis() ;
executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(64);
executor.setRejectedExecutionHandler(retryHandler);
i = 0;
while (i<1000) {
executor.execute(new ComputeRunnable2(i));
i++ ;
}
executor.shutdown();
executor.awaitTermination(60, TimeUnit.SECONDS);
System.out.println("Last " +( System.currentTimeMillis() - start ) +" ms");
} catch (Exception e) {
e.printStackTrace() ;
}
}
}
class ComputeRunnable implements Runnable {
int id = 0 ;
public ComputeRunnable(int i ) {
id = i ;
}
@Override
public void run() {
id =id *9 ;
// System.out.println("ComputeRunnable1 started!");
}
}
class ComputeRunnable2 implements Runnable {
int id = 0 ;
public ComputeRunnable2(int i ) {
id = i ;
}
@Override
public void run() {
id =id *4 ;
// System.out.println("ComputeRunnable2 started!");
}
}
package concurrencyTest;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
public class ConcurrentTest2 {
public static void main(String[] args) {
long start = System.currentTimeMillis() ;
ConcurrentTest2 blt = new ConcurrentTest2() ;
List<Integer> messages = new ArrayList<Integer>();
for(int i =0 ;i<10000; i++) {
messages.add(i) ;
}
ExecutorService pool = Executors.newFixedThreadPool(64); // 并发测试
CompletionService<Boolean> exchangeResult = new ExecutorCompletionService<Boolean>(
pool);
int count = 0 ;
try {
for(Integer i : messages) {
Future future = exchangeResult.submit(blt.new Worker(i));
Object result;
result = future.get();
if (result != null) count++ ;
}
pool.shutdown() ;
pool.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
System.out.println("Last " + (System.currentTimeMillis() - start) + " ms count: " +count);
}
class Worker implements Callable<Boolean> {
Integer msg;
public Worker(Integer msg) {
this.msg = msg;
}
@Override
public Boolean call() throws Exception {
try {
int sum = msg ;
} catch (Exception e) {
e.printStackTrace() ;
}
return true;
}
}
}
Last 228 ms count: 10000
前一种方法 :
package concurrencyTest;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class ConcurrentTest {
public static void main(String[] args) {
long start = System.currentTimeMillis() ;
ConcurrentTest blt = new ConcurrentTest() ;
List<Integer> messages = new ArrayList<Integer>();
for(int i =0 ;i<10000; i++) {
messages.add(i) ;
}
RejectedExecutionHandler retryHandler = new RetryRejectedExecutionHandler();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(64);
executor.setRejectedExecutionHandler(retryHandler);
for (Integer i : messages) {
executor.execute(blt.new Worker(i));
}
executor.shutdown();
try {
executor.awaitTermination(60, TimeUnit.SECONDS);
} catch (Exception e) {
e.printStackTrace() ;
}
System.out.println("Last " + (System.currentTimeMillis() - start) + " ms count: ");
}
class Worker implements Runnable {
Integer msg;
public Worker(Integer msg) {
this.msg = msg;
}
@Override
public void run() {
try {
int sum = msg ;
} catch (Exception e) {
e.printStackTrace() ;
}
}
}
}
class RetryRejectedExecutionHandler implements RejectedExecutionHandler {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace() ;
}
executor.execute(r);
}
}
Last 49
package concurrencyTest;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class ArrayTest {
public static void main(String[] args) {
long start = System.currentTimeMillis() ;
ArrayTest blt = new ArrayTest() ;
List<Integer> messages = new ArrayList<Integer>();
for(int i =0 ;i<10000; i++) {
messages.add(i) ;
ExecutorService pool = Executors.newFixedThreadPool(64); // 并发测试
CompletionService<Boolean> exchangeResult = new ExecutorCompletionService<Boolean>(
pool);
int count = 0 ;
try {
for(Integer i : messages) {
// exchangeResult.submit(blt.new Worker(i));
// Future<Boolean> f = exchangeResult.poll();
Future future = exchangeResult.submit(blt.new Worker(i));
Object result;
result = future.get();
if (result != null) count++ ;
}
pool.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
System.out.println("Last " + (System.currentTimeMillis() - start) + " ms count: " + count);
}
class Worker implements Callable<Boolean> {
Integer msg;
public Worker(Integer msg) {
this.msg = msg;
}
@Override
public Boolean call() throws Exception {
try {
int sum = msg ;
} catch (Exception e) {
e.printStackTrace() ;
}
return true;
}
}
}
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
public class Test{
public static void main(String[] args){
int totalTask = 100;
ExecutorService executor = Executors.newFixedThreadPool(1);
for(int i = 0 ; i < totalTask ; i ++){
//伪代码
Future future = executor.submit(Callable<Object> task);
Object result = future.get();
}
}
}
本身你采用的是CachedThreadPool();你不断的提交任务那么就会不限制的创建线程执行.然后再在另一个while中get所有的结果.
按照这样的逻辑如果本身创建线程太多的话那么系统自然会受不了啦
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.io.IntWritable;
public class ArrayTest {
public static void main(String[] args) {
long start = System.currentTimeMillis() ;
ArrayTest blt = new ArrayTest() ;
List<IntWritable> messages = new ArrayList<IntWritable>();
for(int i =0 ;i<10000; i++) {
messages.add(new IntWritable(i)) ;
}
// RejectedExecutionHandler retryHandler = new RetryRejectedExecutionHandler();
// ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
// .newCachedThreadPool();
// executor.setMaximumPoolSize(64);
// executor.setRejectedExecutionHandler(retryHandler);
//
// for (IntWritable i : messages) {
// executor.execute(blt.new Worker(i));
// }
//
// executor.shutdown();
// try {
// executor.awaitTermination(60, TimeUnit.SECONDS);
// } catch (Exception e) {
// e.printStackTrace() ;
// }
ExecutorService pool = Executors.newCachedThreadPool(); // 并发测试
CompletionService<Boolean> exchangeResult = new ExecutorCompletionService<Boolean>(
pool);
int destSize = 0;
for(IntWritable i : messages) {
exchangeResult.submit(blt.new Worker(i));
destSize++;
}
int count = 0;
while (count < destSize) {
Future<Boolean> f = exchangeResult.poll();
if (f == null)
continue;
count++ ;
}
try {
pool.awaitTermination(60, TimeUnit.SECONDS);
} catch (Throwable e) {
e.printStackTrace();
}
System.out.println("Last " + (System.currentTimeMillis() - start) + " ms");
}
class Worker implements Callable<Boolean> {
IntWritable msg;
public Worker(IntWritable msg) {
this.msg = msg;
}
@Override
public Boolean call() throws Exception {
try {
int sum = msg.get() ;
} catch (Exception e) {
e.printStackTrace() ;
}
return true;
}
}
// class Worker implements Runnable {
// IntWritable msg;
//
// public Worker(IntWritable msg) {
// this.msg = msg;
// }
//
// @Override
// public void run() {
// try {
// int sum = msg.get() ;
// } catch (Exception e) {
// e.printStackTrace() ;
// }
// }
// }
}
第一种: Last 60186 ms
第二段代码:
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.io.IntWritable;
public class ArrayTest {
public static void main(String[] args) {
long start = System.currentTimeMillis() ;
ArrayTest blt = new ArrayTest() ;
List<IntWritable> messages = new ArrayList<IntWritable>();
for(int i =0 ;i<10000; i++) {
messages.add(new IntWritable(i)) ;
}
RejectedExecutionHandler retryHandler = new RetryRejectedExecutionHandler();
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors
.newCachedThreadPool();
executor.setMaximumPoolSize(64);
executor.setRejectedExecutionHandler(retryHandler);
for (IntWritable i : messages) {
executor.execute(blt.new Worker(i));
}
executor.shutdown();
try {
executor.awaitTermination(60, TimeUnit.SECONDS);
} catch (Exception e) {
e.printStackTrace() ;
}
// ExecutorService pool = Executors.newCachedThreadPool(); // 并发测试
// CompletionService<Boolean> exchangeResult = new ExecutorCompletionService<Boolean>(
// pool);
// int destSize = 0;
// for(IntWritable i : messages) {
// exchangeResult.submit(blt.new Worker(i));
// destSize++;
// }
//
// int count = 0;
// while (count < destSize) {
// Future<Boolean> f = exchangeResult.poll();
// if (f == null)
// continue;
// count++ ;
// }
// try {
// pool.awaitTermination(60, TimeUnit.SECONDS);
// } catch (Throwable e) {
// e.printStackTrace();
// }
System.out.println("Last " + (System.currentTimeMillis() - start) + " ms");
}
// class Worker implements Callable<Boolean> {
// IntWritable msg;
//
// public Worker(IntWritable msg) {
// this.msg = msg;
// }
//
// @Override
// public Boolean call() throws Exception {
// try {
// int sum = msg.get() ;
// } catch (Exception e) {
// e.printStackTrace() ;
// }
// return true;
// }
// }
class Worker implements Runnable {
IntWritable msg;
public Worker(IntWritable msg) {
this.msg = msg;
}
@Override
public void run() {
try {
int sum = msg.get() ;
} catch (Exception e) {
e.printStackTrace() ;
}
}
}
}
class RetryRejectedExecutionHandler implements RejectedExecutionHandler {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace() ;
}
executor.execute(r);
}
}
Last 271 ms
可能是前一种写法有问题,导致性能很差,但是奇怪没有报错!第一种写法,即使用 Callable<Boolean>在线程数较少时(<100),没有问题,但是线程数较大,如100,000,则会由于资源不足而阻塞,性能急剧降低。
第二种写法则不存在此问题。