Compare commits
13 commits
Author | SHA1 | Date | |
---|---|---|---|
Gabriel Augendre | 0b2a43128b | ||
Gabriel Augendre | 2f4b560b9b | ||
Gabriel Augendre | 989fe96f8b | ||
Gabriel Augendre | b6ba3d3506 | ||
Gabriel Augendre | 561289614f | ||
Gabriel Augendre | 614fd2fdd7 | ||
Gabriel Augendre | 655616dd1b | ||
Gabriel Augendre | 6ed9532752 | ||
Gabriel Augendre | 30bc840a47 | ||
Gabriel Augendre | b70a3d873a | ||
Gabriel Augendre | a1cc4c98f4 | ||
Gabriel Augendre | ca4804b818 | ||
Gabriel Augendre | 339580e36f |
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -221,3 +221,4 @@ Temporary Items
|
||||||
|
|
||||||
/target
|
/target
|
||||||
**/*.rs.bk
|
**/*.rs.bk
|
||||||
|
.direnv
|
||||||
|
|
2
.mise.toml
Normal file
2
.mise.toml
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
[tools]
|
||||||
|
python = {version="3.11", virtualenv=".venv"}
|
|
@ -19,15 +19,9 @@ repos:
|
||||||
args:
|
args:
|
||||||
- --markdown-linebreak-ext=md
|
- --markdown-linebreak-ext=md
|
||||||
- repo: https://github.com/golangci/golangci-lint
|
- repo: https://github.com/golangci/golangci-lint
|
||||||
rev: v1.42.0
|
rev: v1.54.2
|
||||||
hooks:
|
hooks:
|
||||||
- id: golangci-lint
|
- id: golangci-lint
|
||||||
- repo: https://github.com/TekWizely/pre-commit-golang
|
|
||||||
rev: v1.0.0-beta.4
|
|
||||||
hooks:
|
|
||||||
- id: go-fumpt
|
|
||||||
args:
|
|
||||||
- -w
|
|
||||||
- repo: https://github.com/PyCQA/isort
|
- repo: https://github.com/PyCQA/isort
|
||||||
rev: 5.9.3
|
rev: 5.9.3
|
||||||
hooks:
|
hooks:
|
||||||
|
|
39
LICENSE
39
LICENSE
|
@ -1,24 +1,25 @@
|
||||||
DISCLAIMER: The files under "data/raw_data" are not covered by this license as they
|
This is free and unencumbered software released into the public domain.
|
||||||
were not created by the author of this software.
|
|
||||||
|
|
||||||
MIT License
|
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||||
|
distribute this software, either in source code form or as a compiled
|
||||||
|
binary, for any purpose, commercial or non-commercial, and by any
|
||||||
|
means.
|
||||||
|
|
||||||
Copyright (c) 2021 Gabriel Augendre
|
In jurisdictions that recognize copyright laws, the author or authors
|
||||||
|
of this software dedicate any and all copyright interest in the
|
||||||
|
software to the public domain. We make this dedication for the benefit
|
||||||
|
of the public at large and to the detriment of our heirs and
|
||||||
|
successors. We intend this dedication to be an overt act of
|
||||||
|
relinquishment in perpetuity of all present and future rights to this
|
||||||
|
software under copyright law.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
in the Software without restriction, including without limitation the rights
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||||
furnished to do so, subject to the following conditions:
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
For more information, please refer to <https://unlicense.org>
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
|
|
|
@ -95,3 +95,6 @@ inv release <version_name>
|
||||||
# Data sources
|
# Data sources
|
||||||
|
|
||||||
* https://www.insee.fr/fr/information/2560452, Millésime 2021 : Téléchargement des fichiers, CSV
|
* https://www.insee.fr/fr/information/2560452, Millésime 2021 : Téléchargement des fichiers, CSV
|
||||||
|
|
||||||
|
# Reuse
|
||||||
|
If you do reuse my work, please consider linking back to this repository 🙂
|
|
@ -67,9 +67,10 @@ type InseeData struct {
|
||||||
// NewInseeData generates an InseeData struct, extracting the data into the relevant fields.
|
// NewInseeData generates an InseeData struct, extracting the data into the relevant fields.
|
||||||
// The data is converted to a human-readable format before being stored.
|
// The data is converted to a human-readable format before being stored.
|
||||||
// If a value can't be determined, the corresponding field is generally set to Unknown.
|
// If a value can't be determined, the corresponding field is generally set to Unknown.
|
||||||
|
// It returns an error when the given number isn't 15 characters long.
|
||||||
func NewInseeData(inseeNumber string) (*InseeData, error) {
|
func NewInseeData(inseeNumber string) (*InseeData, error) {
|
||||||
if len(inseeNumber) != 15 {
|
if len(inseeNumber) != 15 {
|
||||||
return nil, fmt.Errorf("le numéro INSEE number must contain 15 characters")
|
return nil, fmt.Errorf("le numéro INSEE doit contenir 15 caractères")
|
||||||
}
|
}
|
||||||
num := inseeNumber
|
num := inseeNumber
|
||||||
departmentCode := num[5:7]
|
departmentCode := num[5:7]
|
||||||
|
@ -147,6 +148,7 @@ func NewInseeData(inseeNumber string) (*InseeData, error) {
|
||||||
|
|
||||||
// IsValid returns true when the insee number is valid and false when not.
|
// IsValid returns true when the insee number is valid and false when not.
|
||||||
// The insee number is valid when it matches its ControlKey.
|
// The insee number is valid when it matches its ControlKey.
|
||||||
|
// It returns an error when the insee number can't be converted to an integer.
|
||||||
func (insee InseeData) IsValid() (bool, error) {
|
func (insee InseeData) IsValid() (bool, error) {
|
||||||
r := strings.NewReplacer(
|
r := strings.NewReplacer(
|
||||||
"2A", "19",
|
"2A", "19",
|
||||||
|
|
2
main.go
2
main.go
|
@ -12,7 +12,7 @@ func main() {
|
||||||
flag.Usage = func() {
|
flag.Usage = func() {
|
||||||
out := flag.CommandLine.Output()
|
out := flag.CommandLine.Output()
|
||||||
fmt.Fprintf(out, "Usage: %s [flags] [numero_insee...]\n", os.Args[0])
|
fmt.Fprintf(out, "Usage: %s [flags] [numero_insee...]\n", os.Args[0])
|
||||||
fmt.Fprintf(out, "\nCe programme décode les informations contenues dans votre numéro INSEE (numéro de sécurité sociale français)")
|
fmt.Fprintf(out, "\nCe programme décode les informations contenues dans votre numéro INSEE (numéro de sécurité sociale français) ")
|
||||||
fmt.Fprintf(out, "et vous les affiche d'une manière lisible et claire.\n")
|
fmt.Fprintf(out, "et vous les affiche d'une manière lisible et claire.\n")
|
||||||
flag.PrintDefaults()
|
flag.PrintDefaults()
|
||||||
fmt.Fprintf(out, "\nLes arguments numero_insee doivent comporter 15 caractères. Il est possible d'en spécifier plusieurs séparés par un espace.\n")
|
fmt.Fprintf(out, "\nLes arguments numero_insee doivent comporter 15 caractères. Il est possible d'en spécifier plusieurs séparés par un espace.\n")
|
||||||
|
|
57
tasks.py
57
tasks.py
|
@ -9,6 +9,7 @@ from invoke import Context, task
|
||||||
|
|
||||||
TARGETS = [
|
TARGETS = [
|
||||||
"darwin/amd64",
|
"darwin/amd64",
|
||||||
|
"darwin/arm64",
|
||||||
"freebsd/386",
|
"freebsd/386",
|
||||||
"freebsd/amd64",
|
"freebsd/amd64",
|
||||||
"freebsd/arm",
|
"freebsd/arm",
|
||||||
|
@ -27,38 +28,37 @@ GITEA_TOKEN = os.getenv("GITEA_TOKEN")
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def test(context):
|
def test(context: Context):
|
||||||
"""Run tests"""
|
"""Run tests"""
|
||||||
context: Context
|
|
||||||
with context.cd(BASE_DIR):
|
with context.cd(BASE_DIR):
|
||||||
context.run(f"go test ./... -race -bench .", echo=True)
|
context.run(f"go test ./... -race .", echo=True)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def clean(context):
|
def clean(context: Context):
|
||||||
"""Clean dist files"""
|
"""Clean dist files"""
|
||||||
context.run(f"rm -rf {DIST_DIR}", echo=True)
|
context.run(f"rm -rf {DIST_DIR}", echo=True)
|
||||||
|
|
||||||
|
|
||||||
@task(pre=[test], post=[clean])
|
@task(pre=[clean, test], post=[clean])
|
||||||
def release(context, version_name):
|
def release(context: Context, version_name):
|
||||||
"""Create & push git tag + build binaries"""
|
"""Create & push git tag + build binaries"""
|
||||||
tag(context, version_name)
|
tag(context, version_name)
|
||||||
binaries = build(context, version_name)
|
binaries = build(context, version_name)
|
||||||
upload(context, version_name, binaries)
|
archives = compress(context, binaries)
|
||||||
|
upload(context, version_name, archives)
|
||||||
|
|
||||||
|
|
||||||
@task(pre=[test])
|
@task(pre=[test])
|
||||||
def tag(context, version_name):
|
def tag(context: Context, version_name):
|
||||||
"""Create & push a git tag"""
|
"""Create & push a git tag"""
|
||||||
context: Context
|
|
||||||
version_name = fix_version_name(version_name)
|
version_name = fix_version_name(version_name)
|
||||||
context.run(f"git tag -a {version_name} -m '{version_name}'", echo=True)
|
context.run(f"git tag -a {version_name} -m '{version_name}'", echo=True)
|
||||||
context.run("git push --follow-tags", echo=True)
|
context.run("git push --follow-tags", echo=True)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def build(context, version_name):
|
def build(context: Context, version_name):
|
||||||
"""Cross-platform build"""
|
"""Cross-platform build"""
|
||||||
version_name = fix_version_name(version_name)
|
version_name = fix_version_name(version_name)
|
||||||
binaries = []
|
binaries = []
|
||||||
|
@ -80,8 +80,27 @@ def build(context, version_name):
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def upload(ctx, version_name, binaries):
|
def compress(context: Context, binaries):
|
||||||
context: Context
|
"""Compress binaries to .tar.gz"""
|
||||||
|
archives = []
|
||||||
|
with ThreadPoolExecutor() as pool:
|
||||||
|
for binary in binaries:
|
||||||
|
binary_name = binary.name
|
||||||
|
archive_path = DIST_DIR / f"{binary_name}.tar.gz"
|
||||||
|
archives.append(archive_path)
|
||||||
|
pool.submit(_compress_single_binary, context, archive_path, binary_name)
|
||||||
|
return archives
|
||||||
|
|
||||||
|
|
||||||
|
def _compress_single_binary(context, archive_path, binary_name):
|
||||||
|
with context.cd(DIST_DIR):
|
||||||
|
context.run(
|
||||||
|
f"tar czf {archive_path} {binary_name} && rm {binary_name}", echo=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def upload(ctx: Context, version_name, upload_files):
|
||||||
version_name = fix_version_name(version_name)
|
version_name = fix_version_name(version_name)
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
if not GITEA_TOKEN:
|
if not GITEA_TOKEN:
|
||||||
|
@ -97,17 +116,17 @@ def upload(ctx, version_name, binaries):
|
||||||
print(f"The draft release has been created at {html_url}")
|
print(f"The draft release has been created at {html_url}")
|
||||||
api_url = resp.get("url") + "/assets"
|
api_url = resp.get("url") + "/assets"
|
||||||
with ThreadPoolExecutor() as pool:
|
with ThreadPoolExecutor() as pool:
|
||||||
for binary in binaries:
|
for upload_file in upload_files:
|
||||||
pool.submit(post_attachment, api_url, binary, session)
|
pool.submit(post_attachment, api_url, upload_file, session)
|
||||||
print(f"All uploads are finished. Update & publish your draft: {html_url}")
|
print(f"All uploads are finished. Update & publish your draft: {html_url}")
|
||||||
|
|
||||||
|
|
||||||
def post_attachment(api_url, binary, session):
|
def post_attachment(api_url, upload_file, session):
|
||||||
binary = Path(binary)
|
upload_file = Path(upload_file)
|
||||||
name = binary.name
|
name = upload_file.name
|
||||||
url = api_url + f"?name={name}"
|
url = api_url + f"?name={name}"
|
||||||
print(f"Uploading {name}...")
|
print(f"Uploading {name}...")
|
||||||
with open(binary, "rb") as f:
|
with open(upload_file, "rb") as f:
|
||||||
res = session.post(url, files={"attachment": f})
|
res = session.post(url, files={"attachment": f})
|
||||||
status_code = res.status_code
|
status_code = res.status_code
|
||||||
if status_code != 201:
|
if status_code != 201:
|
||||||
|
@ -116,7 +135,7 @@ def post_attachment(api_url, binary, session):
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def pre_process(context):
|
def pre_process(context: Context):
|
||||||
"""Pre-process raw data into JSON"""
|
"""Pre-process raw data into JSON"""
|
||||||
files_to_rename = {
|
files_to_rename = {
|
||||||
r"commune.*\.csv": "commune.csv",
|
r"commune.*\.csv": "commune.csv",
|
||||||
|
|
Loading…
Reference in a new issue