.yml
# hadoop
hadoop:
user:
home: /Dev/hadoop-2.6.0
command: /Dev/hadoop-2.6.0/bin/hadoop.cmd
confDir: /Dev/hadoop-2.6.0/etc/hadoop
defaultFS: hdfs://
yarn:
resourceManager:
hadoopMapreduceJobTracker:
hadoopMapreduceJobHistory:
yarnQueueName: default
hive:
connectionTimeout: 1800000
maximumPoolSize: 1
infoUsername:
driverClassName: org.apache.hive.jdbc.HiveDriver
jdbcUrl: jdbc:hive2://.gradle
// hadoop-hive
implementation group: 'org.springframework.data', name: 'spring-data-hadoop', version: '2.2.1.RELEASE'
implementation ('org.apache.hive:hive-jdbc:2.1.0') {
[new Tuple('org.eclipse.jetty.orbit', 'javax.servlet')
, new Tuple('org.eclipse.jetty.aggregate', 'jetty-all')
, new Tuple('org.json', 'json')
, new Tuple('org.slf4j', 'slf4j-log4j12')
, new Tuple('org.apache.logging.log4j', 'log4j-slf4j-impl')
].each {
exclude group: "${it.get(0)}", module: "${it.get(1)}"
}
}
compile group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.11.1'
compile group: 'org.apache.hadoop', 'name': 'hadoop-common', version: '2.6.5'
compile group: 'org.apache.hadoop', 'name': 'hadoop-hdfs', version: '2.6.5'
compile group: 'org.apache.hadoop', 'name': 'hadoop-mapreduce-client-core', version: '2.6.5'
compile group: 'org.apache.hadoop', 'name': 'hadoop-hdfs', version: '2.6.5'
compile group: 'org.apache.hadoop', 'name': 'hadoop-mapreduce-client-core', version: '2.6.5'
// end-hadoop-hive
HadoopHiveConfig.java
package .common.config;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.transaction.managed.ManagedTransactionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
@Slf4j
@Configuration
@MapperScan(basePackages = "", sqlSessionFactoryRef = "hiveSqlSessionFactory", annotationClass = HadoopHiveConfig.HiveMapper.class)
public class HadoopHiveConfig {
@Value("${spring.hadoop.home}")
private String hadoopHome;
@Value("${spring.hadoop.user}")
private String hadoopUser;
@Value("${spring.hadoop.command}")
private String hadoopCommand;
@Value("${spring.hadoop.confDir}")
private String hadoopConfDir;
@Value("${spring.hadoop.defaultFS}")
private String hadoopDefaultFS;
@Value("${spring.hadoop.yarn.resourceManager}")
private String yarnResourceManager;
@Value("${spring.hadoop.yarn.hadoopMapreduceJobTracker}")
private String hadoopMapreduceJobTracker;
@Value("${spring.hadoop.yarn.hadoopMapreduceJobHistory}")
private String hadoopMapreduceJobHistory;
@Value("${spring.hadoop.yarn.yarnQueueName}")
private String yarnQueueName;
@Value("${spring.hive.connectionTimeout}")
private long hiveInfoConnectionTimeout;
@Value("${spring.hive.maximumPoolSize}")
private int hiveInfoMaximumPoolSize;
@Value("${spring.hive.infoUsername}")
private String hiveInfoUsername;
@Value("${spring.hive.driverClassName}")
private String hiveInfoDriverClassName;
@Value("${spring.hive.jdbcUrl}")
private String hiveInfoJdbcUrl;
@Value("${spring.hive.poolName}")
private String hivePoolName;
@Bean(name = "hiveSqlSessionFactory")
public SqlSessionFactory sqlSessionFactory(@Qualifier("hikariHiveDatasource") DataSource hikariHiveDataSource) throws Exception {
SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
sessionFactory.setDataSource(hikariHiveDataSource);
sessionFactory.setMapperLocations(
new PathMatchingResourcePatternResolver().getResources("classpath:mapper_hive/*.xml") );
sessionFactory.setTransactionFactory(new ManagedTransactionFactory());
return sessionFactory.getObject();
}
@Bean(name = "hikariHiveDatasource")
public DataSource dataSource() {
setSystemProperty();
HikariConfig config = getHiveHikariConfig();
return new HikariDataSource(config);
}
private void setSystemProperty() {
System.setProperty("hadoop.home.dir", hadoopHome);
System.setProperty("HADOOP_USER_NAME", hadoopUser);
}
protected HikariConfig getHiveHikariConfig() {
HikariConfig config = new HikariConfig();
config.setPoolName(hivePoolName);
config.setJdbcUrl(hiveInfoJdbcUrl);
config.setDriverClassName(hiveInfoDriverClassName);
config.setUsername(hiveInfoUsername);
config.setMaximumPoolSize(hiveInfoMaximumPoolSize);
config.setConnectionTimeout(hiveInfoConnectionTimeout);
return config;
}
/**
* hdfs conf
* @return
*/
public org.apache.hadoop.conf.Configuration getHdfsConfig() {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
// core-site
conf.set("fs.defaultFS", hadoopDefaultFS);
// hdfs-site
conf.set("dfs.replication", "3");
conf.set("dfs.permissions", "false");
return conf;
}
/**
* hdfs conf + yarn conf
* @return
*/
public org.apache.hadoop.conf.Configuration getYarnConfig() {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(getHdfsConfig());
conf.set("mapreduce.job.user.classpath.first", "true");
conf.set("mapreduce.framework.name", "yarn");
conf.set("yarn.resourcemanager.hostname", yarnResourceManager);
if(hadoopMapreduceJobTracker.length() > 0) {
conf.set("mapreduce.jobtracker.address", hadoopMapreduceJobTracker);
conf.set("mapreduce.jobhistory.address", hadoopMapreduceJobHistory);
conf.set("yarn.app.mapreduce.am.staging-dir", "/tmp/mapred-dic/staging");
// 필요함.
conf.set("mapreduce.jobhistory.webapp.address", "0.0.0.0:50030");
conf.set("mapreduce.jobtracker.http.address", "0.0.0.0:51030");
conf.set("mapreduce.tasktracker.http.address", "0.0.0.0:51060");
}
if(yarnQueueName.length() > 0) {
conf.set("mapreduce.job.queuename", yarnQueueName);
}
return conf;
}
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface HiveMapper {
}
}'java' 카테고리의 다른 글
| java.util.concurrent.CompletionException: java.lang.IllegalStateException: Duplicate key (0) | 2022.03.25 |
|---|---|
| spring-batch 에서 kafka 사용할때 ConcurrentModificationException 발생시 (0) | 2021.11.25 |
| gson 필드에 어노테이션으로 필드 지정 (0) | 2021.09.13 |
| bit AND 연산 구현 (0) | 2021.08.19 |
| from idx to idx 설정 (0) | 2020.04.07 |