BigQueryで不要フィールドを無視する

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

package me.masumi.samples.bq

import com.google.cloud.bigquery.*

private val logger = LogManager.getLogger(javaClass)
val waitOptions: Array<out RetryOption> = null
val bqOptions: BigQueryOptions = null

fun loadFromGS(dest: TableId, srcUri: String, schema: Schema, format: FormatOptions) {
    this.execute(
        LoadJobConfiguration.of(dest, srcUri, format).toBuilder()
            .setCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
            .setIgnoreUnknownValues(true)
            .setSchema(schema).build())
}

fun execute(conf: JobConfiguration) {
    val id = JobId.of(java.util.UUID.randomUUID().toString())
    try {
        val completed = bqOptions.service
                .create(JobInfo.newBuilder(conf).setJobId(id).build())
                .waitFor(*waitOptions)
        if (completed != null) {
            val e = completed.status.error
            if (e == null) {
                logger.info(successMessage(id, conf))
                return
            }
            throw JobException(id, conf, e.message)
        }
        throw JobException(id, conf, "var completed is null")
    } catch (e: InterruptedException) {
        throw JobException(id, conf, e)
    }
}
class JobException : Exception {
    internal constructor(id: JobId, conf: JobConfiguration, msg: String) : super(failureMessage(id, conf, msg))
    internal constructor(id: JobId, conf: JobConfiguration, e: java.lang.Exception) : super(failureMessage(id, conf, e))
}

private fun failureMessage(id: JobId, conf: JobConfiguration, msg: String): String {
    return "${messagePrefix(id, conf)}[FAILURE] ($msg)"
}