ETL常用工具类
2021/5/16 10:25:16
本文主要是介绍ETL常用工具类,对大家解决编程问题具有一定的参考价值,需要的程序猿们随着小编来一起学习吧!
字符相关
public class CharsetUtils { private enum Charset { /** * 7位ASCII字符,也叫作ISO646-US、Unicode字符集的基本拉丁块 */ US_ASCII("US-ASCII", "位ASCII字符,也叫作ISO646-US、Unicode字符集的基本拉丁块 "), ISO_8859_1("ISO-8859-1", "ISO 拉丁字母表 No.1,也叫作 ISO-LATIN-1"), GBK("GBK", "中文超大字符集"), UTF_8("UTF-8", "8 位 UCS 转换格式"), UTF_16BE("UTF-16BE", "16 位 UCS 转换格式,Big Endian(最低地址存放高位字节)字节顺序"), UTF_16LE("UTF_16LE", "16 位 UCS 转换格式,Big Endian(最低地址存放高位字节)字节顺序"), UTF_16("UTF_16", "16 位 UCS 转换格式,字节顺序由可选的字节顺序标记来标识"); private String encode; private String desc; public String getEncode() { return encode; } public void setEncode(String encode) { this.encode = encode; } public String getDesc() { return desc; } public void setDesc(String desc) { this.desc = desc; } private Charset(String encode, String desc) { this.encode = encode; this.desc = desc; } } /** * 获取传入字符串的编码格式 * * @param str * @return */ public static String getEncode(String str) throws UnsupportedEncodingException { if (!StringUtils.isEmpty(str)) { for (Charset charset : Charset.values()) { String tryStr = new String(str.getBytes(charset.getEncode()), charset.getEncode()); if (str.equals(tryStr)) return charset.getEncode(); } } throw new UnsupportedEncodingException("编码库中不存在"); } /** * 字符串编码转换的实现方法 * * @param str 待转换编码的字符串 * @param newCharset 目标编码 * @return * @throws UnsupportedEncodingException */ public static String changeCharset(String str, String newCharset) throws UnsupportedEncodingException { if (str != null) { //获取到原字符编码 String charsetName = getEncode(str); //用默认字符编码解码字符串。 byte[] bs = str.getBytes(charsetName); //用新的字符编码生成字符串 return new String(bs, newCharset); } return null; } public static String changeCharset(String str, String oldCharset, String newCharset) throws UnsupportedEncodingException { if (str != null) { //获取到原字符编码 String charsetName = oldCharset; //用默认字符编码解码字符串。 byte[] bs = str.getBytes(charsetName); //用新的字符编码生成字符串 return new String(bs, newCharset); } return null; } /** * 将字符编码转换成US-ASCII码 */ public static String toASCII(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.US_ASCII.getEncode()); } /** * 将字符编码转换成ISO-8859-1码 */ public static String toISO_8859_1(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.ISO_8859_1.getEncode()); } /** * 将字符编码转换成UTF-8码 */ public static String toUTF_8(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.UTF_8.getEncode()); } /** * 将字符编码转换成UTF-16BE码 */ public static String toUTF_16BE(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.UTF_16BE.getEncode()); } /** * 将字符编码转换成UTF-16LE码 */ public static String toUTF_16LE(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.UTF_16LE.getEncode()); } /** * 将字符编码转换成UTF-16码 */ public static String toUTF_16(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.UTF_16.getEncode()); } /** * 将字符编码转换成GBK码 */ public static String toGBK(String str) throws UnsupportedEncodingException { return changeCharset(str, Charset.GBK.getEncode()); } }
坐标相关
object CoordinateTransformUtil { val x_pi: Double = 3.14159265358979324 * 3000.0 / 180.0 // π val pi = 3.1415926535897932384626 // 长半轴 val a = 6378245.0 // 扁率 val ee = 0.00669342162296594323 /** * 百度坐标系(BD-09)转WGS坐标 * * @param lng 百度坐标纬度 * @param lat 百度坐标经度 * @return WGS84坐标数组 */ def bd09towgs84(lng: Double, lat: Double): Array[Double] = { val gcj = bd09togcj02(lng, lat) val wgs84 = gcj02towgs84(gcj(0), gcj(1)) wgs84 } /** * WGS坐标转百度坐标系(BD-09) * * @param lng WGS84坐标系的经度 * @param lat WGS84坐标系的纬度 * @return 百度坐标数组 */ def wgs84tobd09(lng: Double, lat: Double): Array[Double] = { val gcj = wgs84togcj02(lng, lat) val bd09 = gcj02tobd09(gcj(0), gcj(1)) bd09 } /** * 火星坐标系(GCJ-02)转百度坐标系(BD-09) * <p> * 谷歌、高德——>百度 * * @param lng 火星坐标经度 * @param lat 火星坐标纬度 * @return 百度坐标数组 */ def gcj02tobd09(lng: Double, lat: Double): Array[Double] = { val z = Math.sqrt(lng * lng + lat * lat) + 0.00002 * Math.sin(lat * x_pi) val theta = Math.atan2(lat, lng) + 0.000003 * Math.cos(lng * x_pi) val bd_lng = z * Math.cos(theta) + 0.0065 val bd_lat = z * Math.sin(theta) + 0.006 Array[Double](bd_lng, bd_lat) } /** * 百度坐标系(BD-09)转火星坐标系(GCJ-02) * <p> * 百度——>谷歌、高德 * * @param bd_lon 百度坐标纬度 * @param bd_lat 百度坐标经度 * @return 火星坐标数组 */ def bd09togcj02(bd_lon: Double, bd_lat: Double): Array[Double] = { val x = bd_lon - 0.0065 val y = bd_lat - 0.006 val z = Math.sqrt(x * x + y * y) - 0.00002 * Math.sin(y * x_pi) val theta = Math.atan2(y, x) - 0.000003 * Math.cos(x * x_pi) val gg_lng = z * Math.cos(theta) val gg_lat = z * Math.sin(theta) Array[Double](gg_lng, gg_lat) } /** * WGS84转GCJ02(火星坐标系) * * @param lng WGS84坐标系的经度 * @param lat WGS84坐标系的纬度 * @return 火星坐标数组 */ def wgs84togcj02(lng: Double, lat: Double): Array[Double] = { if (out_of_china(lng, lat)) return Array[Double](lng, lat) var dlat = transformlat(lng - 105.0, lat - 35.0) var dlng = transformlng(lng - 105.0, lat - 35.0) val radlat = lat / 180.0 * pi var magic = Math.sin(radlat) magic = 1 - ee * magic * magic val sqrtmagic = Math.sqrt(magic) dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi) dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi) val mglat = lat + dlat val mglng = lng + dlng Array[Double](mglng, mglat) } /** * GCJ02(火星坐标系)转GPS84 * * @param lng 火星坐标系的经度 * @param lat 火星坐标系纬度 * @return WGS84坐标数组 */ def gcj02towgs84(lng: Double, lat: Double): Array[Double] = { if (out_of_china(lng, lat)) return Array[Double](lng, lat) var dlat = transformlat(lng - 105.0, lat - 35.0) var dlng = transformlng(lng - 105.0, lat - 35.0) val radlat = lat / 180.0 * pi var magic = Math.sin(radlat) magic = 1 - ee * magic * magic val sqrtmagic = Math.sqrt(magic) dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi) dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi) val mglat = lat + dlat val mglng = lng + dlng Array[Double](lng * 2 - mglng, lat * 2 - mglat) } /** * 纬度转换 * * @param lng * @param lat * @return */ def transformlat(lng: Double, lat: Double): Double = { var ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + 0.1 * lng * lat + 0.2 * Math.sqrt(Math.abs(lng)) ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0 ret += (20.0 * Math.sin(lat * pi) + 40.0 * Math.sin(lat / 3.0 * pi)) * 2.0 / 3.0 ret += (160.0 * Math.sin(lat / 12.0 * pi) + 320 * Math.sin(lat * pi / 30.0)) * 2.0 / 3.0 ret } /** * 经度转换 * * @param lng * @param lat * @return */ def transformlng(lng: Double, lat: Double): Double = { var ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + 0.1 * lng * lat + 0.1 * Math.sqrt(Math.abs(lng)) ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0 ret += (20.0 * Math.sin(lng * pi) + 40.0 * Math.sin(lng / 3.0 * pi)) * 2.0 / 3.0 ret += (150.0 * Math.sin(lng / 12.0 * pi) + 300.0 * Math.sin(lng / 30.0 * pi)) * 2.0 / 3.0 ret } /** * 判断是否在国内,不在国内不做偏移 * * @param lng * @param lat * @return */ def out_of_china(lng: Double, lat: Double): Boolean = { if (lng < 72.004 || lng > 137.8347) return true else if (lat < 0.8293 || lat > 55.8271) return true false } }
时间相关
object DateTimeUtils { private val ft = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss") private[bigdata] val millis = (s: String) => { DateTime.parse(s, ft).getMillis / 1000 } private[bigdata] val inHour = (millis: Long, sh: Int, eh: Int) => { val date = new DateTime(millis) val hour = date.hourOfDay.get hour >= sh && hour < eh } //每天的毫秒数 private val day = 1000 * 60 * 60 * 24 //日期字符串的格式 private val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") /** * 返回前一天的 00:00:00 字符串格式 * * @return */ def getLastDayStart: String = format.format(getTodayStartMills - day) /** * 返回前一天的 23:59:59 字符串格式 * * @return */ def getLastDayEnd: String = format.format(getTodayStartMills - 1) /** * 返回当日的 00:00:00 字符串格式 * * @return */ def getTodayStart: String = { val zero = getTodayStartMills format.format(zero) } /** * 返回当日的 23:59:59 字符串格式 * * @return */ def getTodayEnd: String = format.format(getTodayStartMills + day - 1) /** * 返回当日的 00:00:00 毫秒格式 * * @return */ def getTodayStartMills: Long = { val current = System.currentTimeMillis val zero = ((current + TimeZone.getDefault.getRawOffset) / day * day) - TimeZone.getDefault.getRawOffset zero } /** * 返回前一天的 00:00:00 毫秒格式 * * @return */ def getLastDayStartMills: Long = getTodayStartMills - day def main(args: Array[String]): Unit = { val inputPath = "/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/08" // println(path.substring(path.lastIndexOf("/")-7)) // val inputPath="/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/09" // val date = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1) // println(date.toString("yyyyMMdd HH:mm:ss")) val today = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1) val time_tmp = "2019-10-07 23:59:22" //loc_time 定位日期 整型 10 否 unix10位时间戳 val dataTime = DateTime.parse(time_tmp, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")) val isOk = dataTime.getMillis >= today.minusDays(1).getMillis && dataTime.getMillis < today.getMillis println(isOk) } }
加密
object DesUtils { def bytes2hex(bytes: Array[Byte]): String = { val hex = new StringBuilder() for (i <- 0 to bytes.length - 1) { val b = bytes(i) var negative = false if (b < 0) { negative = true } val inte = Math.abs(b) val temp = Integer.toHexString(inte & 0xFF) if (temp.length() == 1) { hex.append("0") } // hex.append(temp.toLowerCase()) hex.append(temp) } hex.toString } // 生成AES密鈅 @throws(classOf[Exception]) def genKeyAES(): String = { val pkey = "123" val kgen = KeyGenerator.getInstance("DES") kgen.init(56, new SecureRandom(pkey.getBytes)) val secretKey = kgen.generateKey val enCodeFormat = secretKey.getEncoded val key = new SecretKeySpec(enCodeFormat, "DES") // val cipher = Cipher.getInstance("DES") // 创建密码器 // val keyGen = KeyGenerator.getInstance("DES") // keyGen.init(56) // val key = keyGen.generateKey() val base64Str = Base64.encodeBase64String(key.getEncoded()) base64Str } @throws(classOf[Exception]) def loadKeyAES(base64Key: String): SecretKey = { val bytes = Base64.decodeBase64(base64Key) val key = new SecretKeySpec(bytes, "DES") return key } def encryt(content: String): Array[Byte] = { encrytAES(content.getBytes(), loadKeyAES(genKeyAES)) // bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES))) } def encryt2Str(content: String): String = { bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES))) } @throws(classOf[Exception]) def encrytAES(source: Array[Byte], key: SecretKey): Array[Byte] = { val cipher = Cipher.getInstance("DES") cipher.init(Cipher.ENCRYPT_MODE, key) cipher.doFinal(source) } def decry(source: Array[Byte]): String = { new String(decryptAES(source, loadKeyAES(genKeyAES))) } // def decry2(source: String): String = { // new String(decryptAES(hexStr2Byte(source), loadKeyAES(genKeyAES))) // } @throws(classOf[Exception]) def decryptAES(source: Array[Byte], key: SecretKey): Array[Byte] = { val cipher = Cipher.getInstance("DES") cipher.init(Cipher.DECRYPT_MODE, key) cipher.doFinal(source) } def main(args: Array[String]): Unit = { val content = "张三" println(decry(encryt(content))) } } object MD5Utils { def encode32(text: String): String = { val md = MessageDigest.getInstance("MD5") md.update(text.getBytes) val b = md.digest var i = 0 val buf = new StringBuffer("") var offset = 0 while ( { offset < b.length }) { i = b(offset) if (i < 0) i += 256 if (i < 16) buf.append("0") buf.append(Integer.toHexString(i)) { offset += 1; offset - 1 } } buf.toString } def encode16(text: String): String = { encode32(text).substring(8, 24) } /** * * 任意文件转换成Md5 * Can convert a text to MD5 * * @param inputStream * @return md5 */ def encode(inputStream: InputStream): String = { var in = inputStream try { val digester = MessageDigest.getInstance("MD5") val bytes = new Array[Byte](8192) var byteCount = 0 while ( { (byteCount = in.read(bytes)); byteCount > 0 }) digester.update(bytes, 0, byteCount) val digest = digester.digest val sb = new StringBuffer for (b <- digest) { val a = b & 0xff var hex = Integer.toHexString(a) if (hex.length == 1) hex = 0 + hex sb.append(hex) } return sb.toString } catch { case e: Exception => e.printStackTrace() } finally if (in != null) { try in.close catch { case e: IOException => e.printStackTrace() } in = null } null } }
GIS计算
object Spatial { private val EARTH_RADIUS = 6378.137 // 地球半径 // 使用经纬度估算两点之间的距离(米) private[bigdata] val distance = (lon1: Float, lat1: Float, lon2: Float, lat2: Float) => { val (l1, l2) = (rad(lat1), rad(lat2)) val a = l1 - l2 val b = rad(lon1) - rad(lon2) val d = EARTH_RADIUS * 2 * math.asin(math.sqrt(math.pow(math.sin(a / 2), 2) + math.cos(l1) * math.cos(l2) * math.pow(math.sin(b / 2), 2))) math.round(d * 10000) / 10 } // 弧度到角度(PI=180°) private def rad(d: Float): Float = d * math.Pi.toFloat / 180 }
其他
object StdUtils { def isCnLngLat(lng: Double, lat: Double): Boolean = { //纬度3.86~53.55,经度73.66~135.05 lng >= 73.66 && lng <= 135.05 && lat >= 3.86 && lat <= 53.55 } def fillTime(date: String, pattern: String = "yyyy-MM-dd HH:mm:ss"): String = { if (StdUtils.isTime(date, Array(pattern))) { return date } var time1 = date if (date.length > 19) { time1 = date.substring(0, 19) return time1 } val time2 = "1971-01-01 00:00:00" time1 + time2.substring(time1.length) } def main(args: Array[String]): Unit = { println(fillTime("2018-10-01 00:02:35.000")) } def isMonth(str: String): Boolean = { val parsePatterns = Array("yyyy/MM", "yyyy-MM", "yyyyMM") return isTime(str, parsePatterns) } // val parsePatterns = Array("yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyyMMdd") def isDate(str: String): Boolean = { val parsePatterns = Array("yyyy/MM/dd", "yyyy-MM-dd", "yyyyMMdd") return isTime(str, parsePatterns) } def isHour(str: String): Boolean = { val parsePatterns = Array("yyyy/MM/dd/HH", "yyyy-MM-dd HH", "yyyyMMdd HH") return isTime(str, parsePatterns) } // def main(args: Array[String]): Unit = { // val res = isTime("20571501040057", Array("yyyyMMddHHmmss")) // println(res) // } def isTime(str: String, parsePatterns: Array[String]): Boolean = { try { val date = DateUtils.parseDate(str, null, parsePatterns: _*) if (date.getTime > DateTime.now().getMillis) { return false; } } catch { case e: Exception => return false } return true; } def replaceBlank(str: String) = { str.replaceAll(" ", "") } /** * 临时替代数据年份 * * @param date * @param year * @return */ def replaceYear(date: String, year: String = "2017"): String = { if (!date.startsWith(year)) { year + date.substring(4) } else { date } } /** * 同济需求 * * @param date * @param yearMonth * @return */ def replaceYearMonth(date: String, yearMonth: String = "201710"): String = { if (date.startsWith("20190312")) { return "20171023" } if (!date.startsWith(yearMonth)) { yearMonth + date.substring(6) } else { date } } // def main(args: Array[String]): Unit = { // val arr = "2939131247,19980077,-15093343,201,2017-01-01 15:34:23,1,0,0,1,0,0,0,0,0,0,0,0,131856,131856,131856,131856,16.987055,16.987055,16.987055,16.987055,866,866,866,866,130896,130896,130896,130896,4.746764,4.746764,4.746764,4.746764,0,0,0,0,0,0,1".split(",") // // println(arr) // println(arr.length) // } /** * 字符串前补0 * * @param fillsize * @param data * @return */ def fillZero(fillsize: Int, data: String): String = { val prefix = new StringBuilder Range(0, fillsize - data.trim.length).foreach(x => { prefix.append("0") }) prefix.append(data.trim).toString() } /** * 判断全不为空 * * @param arr * @return */ def allNotEmpty(arr: Array[String]): Boolean = { var res = true for (e <- arr if res) { res = StringUtils.isNotEmpty(e) } res } }
这篇关于ETL常用工具类的文章就介绍到这儿,希望我们推荐的文章对大家有所帮助,也希望大家多多支持为之网!
- 2024-09-28微服务架构中API版本控制的实践
- 2024-09-28AI给的和自己写的Python代码,都无法改变输入框的内容,替换也不行
- 2024-09-27Sentinel配置限流资料:新手入门教程
- 2024-09-27Sentinel配置限流资料详解
- 2024-09-27Sentinel限流资料:新手入门教程
- 2024-09-26Sentinel限流资料入门详解
- 2024-09-26Springboot框架资料:初学者入门教程
- 2024-09-26Springboot框架资料详解:新手入门教程
- 2024-09-26Springboot企业级开发资料:新手入门指南
- 2024-09-26SpringBoot企业级开发资料新手指南