Compare commits
27 commits
Author | SHA1 | Date | |
---|---|---|---|
0b2a43128b | |||
2f4b560b9b | |||
989fe96f8b | |||
b6ba3d3506 | |||
561289614f | |||
614fd2fdd7 | |||
655616dd1b | |||
6ed9532752 | |||
30bc840a47 | |||
b70a3d873a | |||
a1cc4c98f4 | |||
ca4804b818 | |||
339580e36f | |||
4773950a65 | |||
2e4bf89368 | |||
c143a64bca | |||
ec024a0189 | |||
c01b54ef50 | |||
9ea22d8c4a | |||
ccce40a90d | |||
4465be80ea | |||
46bd3cdbf5 | |||
9f626c8450 | |||
47b6a8b958 | |||
b2bc5e0d1c | |||
2cc87d878f | |||
bf1f57eb00 |
16 changed files with 339 additions and 81 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -1,3 +1,4 @@
|
|||
insee_number_translator
|
||||
|
||||
# Created by https://www.gitignore.io/api/pycharm,python
|
||||
# Edit at https://www.gitignore.io/?templates=pycharm,python
|
||||
|
@ -220,3 +221,4 @@ Temporary Items
|
|||
|
||||
/target
|
||||
**/*.rs.bk
|
||||
.direnv
|
||||
|
|
2
.mise.toml
Normal file
2
.mise.toml
Normal file
|
@ -0,0 +1,2 @@
|
|||
[tools]
|
||||
python = {version="3.11", virtualenv=".venv"}
|
|
@ -3,31 +3,30 @@ repos:
|
|||
rev: v4.0.1
|
||||
hooks:
|
||||
- id: check-ast
|
||||
types: [python]
|
||||
- id: check-json
|
||||
types: [json]
|
||||
- id: check-toml
|
||||
types: [toml]
|
||||
- id: check-xml
|
||||
types: [xml]
|
||||
- id: check-yaml
|
||||
types: [yaml]
|
||||
- id: end-of-file-fixer
|
||||
exclude: ^data/(curated|raw)_data
|
||||
- id: check-merge-conflict
|
||||
- id: pretty-format-json
|
||||
args:
|
||||
- --autofix
|
||||
- --no-sort-keys
|
||||
exclude: ^data/(curated|raw)_data
|
||||
- id: trailing-whitespace
|
||||
args:
|
||||
- --markdown-linebreak-ext=md
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v1.54.2
|
||||
hooks:
|
||||
- id: golangci-lint
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: 5.9.3
|
||||
hooks:
|
||||
- id: isort
|
||||
types: [python]
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 21.7b0
|
||||
hooks:
|
||||
- id: black
|
||||
types: [python]
|
||||
|
|
39
LICENSE
39
LICENSE
|
@ -1,24 +1,25 @@
|
|||
DISCLAIMER: The files under "data/raw_data" are not covered by this license as they
|
||||
were not created by the author of this software.
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
MIT License
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
Copyright (c) 2021 Gabriel Augendre
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
For more information, please refer to <https://unlicense.org>
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
79
README.md
79
README.md
|
@ -1,18 +1,69 @@
|
|||
# INSEE number translator
|
||||
# INSEE number translator (Français)
|
||||
(English below)
|
||||
Extrait les données des numéros INSEE (numéro de sécurité sociale française).
|
||||
|
||||
Extract data from INSEE number (France)
|
||||
Le numéro de sécurité sociale français est *signifiant*. Cela veut dire que ce n'est pas un numéro aléatoire,
|
||||
il porte des informations.
|
||||
|
||||
* https://www.insee.fr/fr/metadonnees/definition/c1409
|
||||
* https://fr.wikipedia.org/wiki/Num%C3%A9ro_de_s%C3%A9curit%C3%A9_sociale_en_France
|
||||
|
||||
## Commencer
|
||||
### Utiliser un programme prêt à l'emploi
|
||||
Rendez-vous sur la [dernière version](https://git.augendre.info/gaugendre/insee_number_translator/releases/latest)
|
||||
et téléchargez le fichier correspondant à votre système d'exploitation et à votre architecture.
|
||||
Si vous ne savez pas de quoi il s'agit, essayez d'abord `amd64` puis `386` si le premier ne fonctionne pas.
|
||||
Pour macOS, utilisez le fichier `darwin`.
|
||||
|
||||
Ensuite, utilisez la ligne de commande pour exécuter le fichier :
|
||||
```shell
|
||||
./insee 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
```
|
||||
|
||||
Rassurez-vous, ce programme ne transmet aucune information sur le réseau. Toutes les données
|
||||
nécessaires à son fonctionnement sont comprises dans le fichier que vous venez de télécharger.
|
||||
Votre numéro de sécurité sociale ne quitte pas votre ordinateur.
|
||||
|
||||
### Depuis les sources
|
||||
Requiert les outils `go` (https://golang.org/).
|
||||
|
||||
#### Exécution directe
|
||||
```shell
|
||||
go run . 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
```
|
||||
|
||||
#### Compilation et installation
|
||||
```shell
|
||||
go install
|
||||
insee_number_translator 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
```
|
||||
|
||||
# INSEE number translator (English)
|
||||
Extract data from INSEE number (France).
|
||||
|
||||
The french social security number is significant. That means it's not a random number, it bears information.
|
||||
|
||||
* (fr) https://www.insee.fr/fr/metadonnees/definition/c1409
|
||||
* (fr) https://fr.wikipedia.org/wiki/Num%C3%A9ro_de_s%C3%A9curit%C3%A9_sociale_en_France
|
||||
|
||||
## Getting started
|
||||
### Using a pre-built binary
|
||||
Go to the [latest release](https://git.augendre.info/gaugendre/insee_number_translator/releases/latest)
|
||||
and download the binary matching your OS and architecture.
|
||||
and download the binary matching your OS and architecture.
|
||||
If you don't know what that means, try `amd64` first, then `386` if it doesn't work.
|
||||
For macOS, choose `darwin`.
|
||||
|
||||
Then, run the tool via the command line:
|
||||
```shell
|
||||
./insee 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
```
|
||||
Rest assured, this program doesn't transmit any information on the network. All the data
|
||||
necessary for its operation are included in the file you just downloaded.
|
||||
Your social security number doesn't leave your computer.
|
||||
|
||||
### From sources
|
||||
Requires the `go` toolchain (https://golang.org/).
|
||||
|
||||
#### Run
|
||||
```shell
|
||||
go run . 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
|
@ -24,6 +75,26 @@ go install
|
|||
insee_number_translator 269059913116714 168127982980507 299122A00498723 299129742398791 144089943287340
|
||||
```
|
||||
|
||||
## Data sources
|
||||
# Tinker
|
||||
## Update data
|
||||
You can easily update the data by downloading the CSV files available using the link in `Data sources` below.
|
||||
Then, unzip the file in `data/raw_data` and run the following command:
|
||||
|
||||
```shell
|
||||
invoke pre-process
|
||||
```
|
||||
|
||||
This requires [`invoke`](https://www.pyinvoke.org/) on your machine (run `pip install -r requirements.txt`).
|
||||
|
||||
## Release
|
||||
Run tests, create a tag and build binaries:
|
||||
```
|
||||
inv release <version_name>
|
||||
```
|
||||
|
||||
# Data sources
|
||||
|
||||
* https://www.insee.fr/fr/information/2560452, Millésime 2021 : Téléchargement des fichiers, CSV
|
||||
|
||||
# Reuse
|
||||
If you do reuse my work, please consider linking back to this repository 🙂
|
|
@ -1,3 +1,5 @@
|
|||
// Package data is used to parse the insee number into a usable and human-readable data structure.
|
||||
// All the required data is embedded in the package so no external file is required.
|
||||
package data
|
||||
|
||||
import (
|
||||
|
@ -9,9 +11,16 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
var Male = "homme"
|
||||
var Female = "femme"
|
||||
var Unknown = "inconnu(e)"
|
||||
var (
|
||||
// Male is one of the possible values for InseeData.Gender.
|
||||
Male = "homme"
|
||||
// Female is the other possible value for InseeData.Gender.
|
||||
Female = "femme"
|
||||
// Unknown is a value used when determining the value of a string field in InseeData was not possible.
|
||||
Unknown = "inconnu(e)"
|
||||
// France is the only item in the default slice for InseeData.Countries.
|
||||
France = "FRANCE"
|
||||
)
|
||||
|
||||
//go:embed curated_data/countries.json
|
||||
var rawCountries []byte
|
||||
|
@ -22,25 +31,46 @@ var rawDepartments []byte
|
|||
//go:embed curated_data/cities.json
|
||||
var rawCities []byte
|
||||
|
||||
// InseeData contains human-readable data about the insee number used to construct it.
|
||||
type InseeData struct {
|
||||
InseeNumber string `json:"insee_number"`
|
||||
Gender string `json:"gender"`
|
||||
Year int `json:"year"`
|
||||
Month time.Month `json:"month"`
|
||||
Department string `json:"department"`
|
||||
City string `json:"city"`
|
||||
CityCode string `json:"city_code"`
|
||||
Foreign bool `json:"foreign"`
|
||||
Countries []string `json:"countries"`
|
||||
CountryCode string `json:"country_code"`
|
||||
Continent string `json:"continent"`
|
||||
OrderOfBirth int `json:"order_of_birth"`
|
||||
ControlKey int `json:"control_key"`
|
||||
// InseeNumber is the raw number, as given.
|
||||
InseeNumber string `json:"insee_number"`
|
||||
// Gender is either Male or Female.
|
||||
Gender string `json:"gender"`
|
||||
// Year of birth.
|
||||
Year int `json:"year"`
|
||||
// Month of birth.
|
||||
Month time.Month `json:"month"`
|
||||
// Department of birth, represented with its name.
|
||||
Department string `json:"department"`
|
||||
// City of birth, represented with its name.
|
||||
City string `json:"city"`
|
||||
// CityCode is the INSEE code of the City of birth.
|
||||
CityCode string `json:"city_code"`
|
||||
// Foreign is false if the person is born in France, true otherwise.
|
||||
Foreign bool `json:"foreign"`
|
||||
// Countries is the list of country names matching the CountryCode.
|
||||
// Some country codes may match multiple countries, so Countries is a slice.
|
||||
// This is always set to `{"FRANCE"}` when Foreign is false.
|
||||
Countries []string `json:"countries"`
|
||||
// CountryCode is the code of the birth country.
|
||||
CountryCode string `json:"country_code"`
|
||||
// Continent of birth.
|
||||
Continent string `json:"continent"`
|
||||
// OrderOfBirth is the order of birth of the person in the city or country (if Foreign) of birth at the year/month of birth.
|
||||
// For example, 384 would mean that the person is the 384th born in the specific city/country on the given year/month.
|
||||
OrderOfBirth int `json:"order_of_birth"`
|
||||
// ControlKey is the complement to 97 of the insee number (minus the last two digits) modulo 97.
|
||||
ControlKey int `json:"control_key"`
|
||||
}
|
||||
|
||||
// NewInseeData generates an InseeData struct, extracting the data into the relevant fields.
|
||||
// The data is converted to a human-readable format before being stored.
|
||||
// If a value can't be determined, the corresponding field is generally set to Unknown.
|
||||
// It returns an error when the given number isn't 15 characters long.
|
||||
func NewInseeData(inseeNumber string) (*InseeData, error) {
|
||||
if len(inseeNumber) != 15 {
|
||||
return nil, fmt.Errorf("provided insee number must contain 15 characters")
|
||||
return nil, fmt.Errorf("le numéro INSEE doit contenir 15 caractères")
|
||||
}
|
||||
num := inseeNumber
|
||||
departmentCode := num[5:7]
|
||||
|
@ -55,7 +85,7 @@ func NewInseeData(inseeNumber string) (*InseeData, error) {
|
|||
}
|
||||
var city string
|
||||
var department string
|
||||
countries_ := []string{"FRANCE"}
|
||||
countries_ := []string{France}
|
||||
countryCode := ""
|
||||
continent := "Europe"
|
||||
foreign := (dep >= 91 && dep <= 96) || dep == 99
|
||||
|
@ -116,6 +146,9 @@ func NewInseeData(inseeNumber string) (*InseeData, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// IsValid returns true when the insee number is valid and false when not.
|
||||
// The insee number is valid when it matches its ControlKey.
|
||||
// It returns an error when the insee number can't be converted to an integer.
|
||||
func (insee InseeData) IsValid() (bool, error) {
|
||||
r := strings.NewReplacer(
|
||||
"2A", "19",
|
||||
|
@ -130,6 +163,7 @@ func (insee InseeData) IsValid() (bool, error) {
|
|||
return code == insee.ControlKey, nil
|
||||
}
|
||||
|
||||
// String returns a string representation of the InseeData in a human-readable format, suited for printing to stdout.
|
||||
func (insee InseeData) String() string {
|
||||
var result []string
|
||||
result = append(result, insee.InseeNumber)
|
||||
|
@ -153,7 +187,6 @@ func (insee InseeData) String() string {
|
|||
}
|
||||
line = fmt.Sprintf("Vous êtes %s %s, %s en %s probablement en %d.", one, insee.Gender, born, frenchMonth(insee.Month), insee.Year)
|
||||
result = append(result, line)
|
||||
line = ""
|
||||
var zoneType string
|
||||
if insee.Foreign {
|
||||
zoneType = "ce pays"
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewInseeData_ValidForeign(t *testing.T) {
|
||||
|
@ -170,3 +171,13 @@ func TestNewInseeData_ValidFrenchCorsica(t *testing.T) {
|
|||
assert.Equal([]string{"FRANCE"}, insee.Countries)
|
||||
assert.Equal(23, insee.ControlKey)
|
||||
}
|
||||
|
||||
var inseeResult *InseeData
|
||||
|
||||
func BenchmarkNewInseeData(b *testing.B) {
|
||||
var in *InseeData
|
||||
for i := 0; i < b.N; i++ {
|
||||
in, _ = NewInseeData("299122A00498723")
|
||||
}
|
||||
inseeResult = in
|
||||
}
|
||||
|
|
Can't render this file because it is too large.
|
8
go.mod
8
go.mod
|
@ -1,5 +1,9 @@
|
|||
module insee_number_translator
|
||||
module git.augendre.info/gaugendre/insee_number_translator
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/stretchr/testify v1.7.0
|
||||
require (
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
)
|
||||
|
|
8
go.sum
8
go.sum
|
@ -1,11 +1,17 @@
|
|||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
21
main.go
21
main.go
|
@ -3,20 +3,29 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"insee_number_translator/data"
|
||||
"os"
|
||||
|
||||
"git.augendre.info/gaugendre/insee_number_translator/data"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var preProcess bool
|
||||
flag.BoolVar(&preProcess, "pre-process", false, "Pre-process data files.")
|
||||
flag.Usage = func() {
|
||||
out := flag.CommandLine.Output()
|
||||
fmt.Fprintf(out, "Usage: %s [flags] [numero_insee...]\n", os.Args[0])
|
||||
fmt.Fprintf(out, "\nCe programme décode les informations contenues dans votre numéro INSEE (numéro de sécurité sociale français) ")
|
||||
fmt.Fprintf(out, "et vous les affiche d'une manière lisible et claire.\n")
|
||||
flag.PrintDefaults()
|
||||
fmt.Fprintf(out, "\nLes arguments numero_insee doivent comporter 15 caractères. Il est possible d'en spécifier plusieurs séparés par un espace.\n")
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if preProcess {
|
||||
data.PreProcessRawData("data/raw_data", "data/curated_data")
|
||||
numbers := flag.Args()
|
||||
if len(numbers) == 0 {
|
||||
flag.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
numbers := flag.Args()
|
||||
for _, number := range numbers {
|
||||
insee, err := data.NewInseeData(number)
|
||||
if err != nil {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package data
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
|
@ -8,27 +8,33 @@ import (
|
|||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("This is intended for tinkerers only, not for end users.")
|
||||
PreProcessRawData("data/raw_data", "data/curated_data")
|
||||
}
|
||||
|
||||
func PreProcessRawData(sourceFolder, targetFolder string) {
|
||||
err := os.MkdirAll(targetFolder, 0755)
|
||||
err := os.MkdirAll(targetFolder, 0o755)
|
||||
if err != nil {
|
||||
fmt.Printf("Error, couldn't create target folder %s: %s", targetFolder, err)
|
||||
return
|
||||
}
|
||||
err = preProcessCities(sourceFolder+"/commune2021.csv", targetFolder+"/cities.json")
|
||||
err = preProcessCities(sourceFolder+"/commune.csv", targetFolder+"/cities.json")
|
||||
if err != nil {
|
||||
fmt.Println("Error during cities pre processing")
|
||||
fmt.Println(err.Error())
|
||||
fmt.Fprintln(os.Stderr, "Error during cities pre processing")
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
}
|
||||
err = preProcessDepartments(sourceFolder+"/departement2021.csv", targetFolder+"/departments.json")
|
||||
err = preProcessDepartments(sourceFolder+"/departement.csv", targetFolder+"/departments.json")
|
||||
if err != nil {
|
||||
fmt.Println("Error during departments pre processing")
|
||||
fmt.Println(err.Error())
|
||||
fmt.Fprintln(os.Stderr, "Error during departments pre processing")
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
}
|
||||
err = preProcessCountries(sourceFolder+"/pays2021.csv", targetFolder+"/countries.json")
|
||||
err = preProcessCountries(sourceFolder+"/pays.csv", targetFolder+"/countries.json")
|
||||
if err != nil {
|
||||
fmt.Println("Error during countries pre processing")
|
||||
fmt.Println(err.Error())
|
||||
fmt.Fprintln(os.Stderr, "Error during countries pre processing")
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
}
|
||||
fmt.Println("Finished pre-processing data.")
|
||||
}
|
||||
|
||||
func preProcessCities(sourceFileName, targetFileName string) error {
|
||||
|
@ -83,7 +89,7 @@ func preProcessSimpleFile(sourceFileName, targetFileName, codeColumn string) err
|
|||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(targetFileName, itemsJson, 0644)
|
||||
err = ioutil.WriteFile(targetFileName, itemsJson, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -140,7 +146,7 @@ func preProcessCountries(sourceFileName, targetFileName string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(targetFileName, citiesJson, 0644)
|
||||
err = ioutil.WriteFile(targetFileName, citiesJson, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
2
requirements.txt
Normal file
2
requirements.txt
Normal file
|
@ -0,0 +1,2 @@
|
|||
invoke
|
||||
requests
|
128
tasks.py
128
tasks.py
|
@ -1,10 +1,15 @@
|
|||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
from invoke import Context, task
|
||||
|
||||
TARGETS = [
|
||||
"darwin/amd64",
|
||||
"darwin/arm64",
|
||||
"freebsd/386",
|
||||
"freebsd/amd64",
|
||||
"freebsd/arm",
|
||||
|
@ -17,31 +22,138 @@ TARGETS = [
|
|||
"windows/amd64",
|
||||
"windows/arm",
|
||||
]
|
||||
BASE_DIR = Path(__file__).parent.resolve(strict=True)
|
||||
DIST_DIR = BASE_DIR / "dist"
|
||||
GITEA_TOKEN = os.getenv("GITEA_TOKEN")
|
||||
|
||||
|
||||
@task
|
||||
def tag(context, tag):
|
||||
def test(context: Context):
|
||||
"""Run tests"""
|
||||
with context.cd(BASE_DIR):
|
||||
context.run(f"go test ./... -race .", echo=True)
|
||||
|
||||
|
||||
@task
|
||||
def clean(context: Context):
|
||||
"""Clean dist files"""
|
||||
context.run(f"rm -rf {DIST_DIR}", echo=True)
|
||||
|
||||
|
||||
@task(pre=[clean, test], post=[clean])
|
||||
def release(context: Context, version_name):
|
||||
"""Create & push git tag + build binaries"""
|
||||
tag(context, version_name)
|
||||
binaries = build(context, version_name)
|
||||
archives = compress(context, binaries)
|
||||
upload(context, version_name, archives)
|
||||
|
||||
|
||||
@task(pre=[test])
|
||||
def tag(context: Context, version_name):
|
||||
"""Create & push a git tag"""
|
||||
context: Context
|
||||
context.run(f"git tag -a {tag} -m '{tag}'")
|
||||
context.run("git push --follow-tags")
|
||||
version_name = fix_version_name(version_name)
|
||||
context.run(f"git tag -a {version_name} -m '{version_name}'", echo=True)
|
||||
context.run("git push --follow-tags", echo=True)
|
||||
|
||||
|
||||
@task
|
||||
def build(context, version_name):
|
||||
def build(context: Context, version_name):
|
||||
"""Cross-platform build"""
|
||||
version_name = fix_version_name(version_name)
|
||||
binaries = []
|
||||
with ThreadPoolExecutor() as pool:
|
||||
for target in TARGETS:
|
||||
os, arch = target.split("/")
|
||||
binary_name = f"insee-{version_name}-{os}-{arch}"
|
||||
if os == "windows":
|
||||
binary_name += ".exe"
|
||||
binary_path = (
|
||||
Path(__file__).resolve(strict=True).parent / "dist" / binary_name
|
||||
)
|
||||
binary_path = DIST_DIR / binary_name
|
||||
binaries.append(binary_path)
|
||||
pool.submit(
|
||||
context.run,
|
||||
f"go build -o {binary_path}",
|
||||
env={"GOOS": os, "GOARCH": arch},
|
||||
echo=True,
|
||||
)
|
||||
return binaries
|
||||
|
||||
|
||||
@task
|
||||
def compress(context: Context, binaries):
|
||||
"""Compress binaries to .tar.gz"""
|
||||
archives = []
|
||||
with ThreadPoolExecutor() as pool:
|
||||
for binary in binaries:
|
||||
binary_name = binary.name
|
||||
archive_path = DIST_DIR / f"{binary_name}.tar.gz"
|
||||
archives.append(archive_path)
|
||||
pool.submit(_compress_single_binary, context, archive_path, binary_name)
|
||||
return archives
|
||||
|
||||
|
||||
def _compress_single_binary(context, archive_path, binary_name):
|
||||
with context.cd(DIST_DIR):
|
||||
context.run(
|
||||
f"tar czf {archive_path} {binary_name} && rm {binary_name}", echo=True
|
||||
)
|
||||
|
||||
|
||||
@task
|
||||
def upload(ctx: Context, version_name, upload_files):
|
||||
version_name = fix_version_name(version_name)
|
||||
session = requests.Session()
|
||||
if not GITEA_TOKEN:
|
||||
raise ValueError("You need to set the GITEA_TOKEN env var before uploading")
|
||||
session.headers["Authorization"] = f"token {GITEA_TOKEN}"
|
||||
url = "https://git.augendre.info/api/v1/repos/gaugendre/insee_number_translator/releases"
|
||||
resp = session.post(
|
||||
url, json={"name": version_name, "tag_name": version_name, "draft": True}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
resp = resp.json()
|
||||
html_url = resp.get("html_url")
|
||||
print(f"The draft release has been created at {html_url}")
|
||||
api_url = resp.get("url") + "/assets"
|
||||
with ThreadPoolExecutor() as pool:
|
||||
for upload_file in upload_files:
|
||||
pool.submit(post_attachment, api_url, upload_file, session)
|
||||
print(f"All uploads are finished. Update & publish your draft: {html_url}")
|
||||
|
||||
|
||||
def post_attachment(api_url, upload_file, session):
|
||||
upload_file = Path(upload_file)
|
||||
name = upload_file.name
|
||||
url = api_url + f"?name={name}"
|
||||
print(f"Uploading {name}...")
|
||||
with open(upload_file, "rb") as f:
|
||||
res = session.post(url, files={"attachment": f})
|
||||
status_code = res.status_code
|
||||
if status_code != 201:
|
||||
res = res.json()
|
||||
print(f"Status != 201 for {name}: {status_code} {res}")
|
||||
|
||||
|
||||
@task
|
||||
def pre_process(context: Context):
|
||||
"""Pre-process raw data into JSON"""
|
||||
files_to_rename = {
|
||||
r"commune.*\.csv": "commune.csv",
|
||||
r"departement.*\.csv": "departement.csv",
|
||||
r"pays.*\.csv": "pays.csv",
|
||||
}
|
||||
raw_data_dir = BASE_DIR / "data" / "raw_data"
|
||||
for file in raw_data_dir.iterdir():
|
||||
for reg, target_name in files_to_rename.items():
|
||||
reg = re.compile(reg)
|
||||
if reg.match(file.name):
|
||||
file.rename(raw_data_dir / target_name)
|
||||
|
||||
with context.cd(BASE_DIR):
|
||||
context.run("go run ./pre_process")
|
||||
|
||||
|
||||
def fix_version_name(version_name: str):
|
||||
if not version_name.startswith("v"):
|
||||
return f"v{version_name}"
|
||||
return version_name
|
||||
|
|
Loading…
Add table
Reference in a new issue