Skip to content

Commit 23ee62f

Browse files
huumnekzyis
huumn
and
ekzyis
authored
add sndev shell script and enhance docker compose local dev
* add hot reloading worker:dev script * refine docker config * sndev bash script and docker reliability stuff * make posix shell * restart: always -> unless-stopped * proper check for postgres health * add db seed to sndev * refinements after fresh builds * begin adding regtest network * add changes to .env.sample * reorganize docker and add static certs/macroon to lnd * copy wallet and macaroon dbs for deterministic wallets/macaroons * fix perms of shared directories * allow debian useradd with duplicate id * add auto-mining * make bitcoin health check dependent on blockheight * open channel between ln nodes * improve channel opens * add sndev payinvoice * add sndev withdraw * ascii art * add sndev status * sndev passthrough to docker and containers * add sndev psql command * remove script logging * small script cleanup * smaller db seed * pin opensearch version Co-authored-by: ekzyis <[email protected]> * pin opensearch dashboard Co-authored-by: ekzyis <[email protected]> * add sndev prisma * add help for all commands * set -e * s3 and image proxy with broken name resolution * finally fully working image uploads * use a better diff algo --------- Co-authored-by: ekzyis <[email protected]>
1 parent 179a539 commit 23ee62f

33 files changed

+52277
-198
lines changed

‎.env.sample‎

Lines changed: 87 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,22 @@
44
############################################################################
55

66
# github
7-
GITHUB_ID=<YOUR GITHUB ID>
8-
GITHUB_SECRET=<YOUR GITHUB SECRET>
7+
GITHUB_ID=
8+
GITHUB_SECRET=
99

1010
# twitter
11-
TWITTER_ID=<YOUR TWITTER ID>
12-
TWITTER_SECRET=<YOUR TWITTER SECRET>
11+
TWITTER_ID=
12+
TWITTER_SECRET=
1313

1414
# email
15-
LOGIN_EMAIL_SERVER=smtp://<YOUR EMAIL>:<YOUR PASSWORD>@<YOUR SMTP DOMAIN>:587
16-
LOGIN_EMAIL_FROM=<YOUR FROM ALIAS>
15+
LOGIN_EMAIL_SERVER=
16+
LOGIN_EMAIL_FROM=
1717
LIST_MONK_AUTH=
1818

19-
#####################################################################
20-
# OTHER / OPTIONAL #
21-
# configuration for push notifications, slack and imgproxy are here #
22-
#####################################################################
19+
########################################################
20+
# OTHER / OPTIONAL #
21+
# configuration for push notifications, slack are here #
22+
########################################################
2323

2424
# VAPID for Web Push
2525
VAPID_MAILTO=
@@ -30,38 +30,13 @@ VAPID_PRIVKEY=
3030
SLACK_BOT_TOKEN=
3131
SLACK_CHANNEL_ID=
3232

33-
# imgproxy
34-
NEXT_PUBLIC_IMGPROXY_URL=
35-
IMGPROXY_KEY=
36-
IMGPROXY_SALT=
37-
38-
# search
39-
OPENSEARCH_URL=http://opensearch:9200
40-
OPENSEARCH_USERNAME=
41-
OPENSEARCH_PASSWORD=
42-
OPENSEARCH_INDEX=item
43-
OPENSEARCH_MODEL_ID=
33+
# lnurl ... you'll need a tunnel to localhost:3000 for these
34+
LNAUTH_URL=
35+
LNWITH_URL=
4436

45-
#######################################################
46-
# WALLET / OPTIONAL #
47-
# if you want to work with payments you'll need these #
48-
#######################################################
49-
50-
# lnd
51-
LND_CERT=<YOUR LND HEX CERT>
52-
LND_MACAROON=<YOUR LND HEX MACAROON>
53-
LND_SOCKET=<YOUR LND GRPC HOST>:<YOUR LND GRPC PORT>
54-
55-
# lnurl
56-
LNAUTH_URL=<PUBLIC URL TO /api/lnauth>
57-
LNWITH_URL=<PUBLIC URL TO /api/lnwith>
58-
59-
# nostr (NIP-57 zap receipts)
60-
NOSTR_PRIVATE_KEY=<YOUR NOSTR PRIVATE KEY IN HEX>
61-
62-
###############
63-
# LEAVE AS IS #
64-
###############
37+
#########################
38+
# SNDEV STUFF WE PRESET #
39+
#########################
6540

6641
# static things
6742
NEXTAUTH_URL=http://localhost:3000/api/auth
@@ -72,7 +47,21 @@ NEXTAUTH_SECRET=3_0W_PhDRZVanbeJsZZGIEljexkKoGbL6qGIqSwTjjI
7247
JWT_SIGNING_PRIVATE_KEY={"kty":"oct","kid":"FvD__hmeKoKHu2fKjUrWbRKfhjimIM4IKshyrJG4KSM","alg":"HS512","k":"3_0W_PhDRZVanbeJsZZGIEljexkKoGbL6qGIqSwTjjI"}
7348
INVOICE_HMAC_KEY=a4c1d9c81edb87b79d28809876a18cf72293eadb39f92f3f4f2f1cfbdf907c91
7449

75-
# imgproxy options
50+
# lnd
51+
# xxd -p -c0 docker/lnd/sn/regtest/admin.macaroon
52+
LND_CERT=2d2d2d2d2d424547494e2043455254494649434154452d2d2d2d2d0a4d494943516a43434165696741774942416749516139493834682b48653350385a437541525854554d54414b42676771686b6a4f50515144416a41344d5238770a485159445651514b45785a73626d5167595856306232646c626d56795958526c5a43426a5a584a304d5255774577594456515144457778694e6a41785a5749780a4d474d354f444d774868634e4d6a51774d7a41334d5463774d6a45355768634e4d6a55774e5441794d5463774d6a4535576a41344d523877485159445651514b0a45785a73626d5167595856306232646c626d56795958526c5a43426a5a584a304d5255774577594456515144457778694e6a41785a5749784d474d354f444d770a5754415442676371686b6a4f5051494242676771686b6a4f50514d4242774e4341415365596a4b62542b4a4a4a37624b6770677a6d6c3278496130364e3174680a2f4f7033533173382b4f4a41387836647849682f326548556b4f7578675a36703549434b496f375a544c356a5963764375793941334b6e466f3448544d4948510a4d41344741315564447745422f775145417749437044415442674e56485355454444414b4267677242674546425163444154415042674e5648524d42416638450a425441444151482f4d4230474131556444675157424252545756796e653752786f747568717354727969466d6a36736c557a423542674e5648524545636a42770a676778694e6a41785a5749784d474d354f444f4343577876593246736147397a64494947633235666247356b6768526f62334e304c6d52765932746c636935700a626e526c636d356862494945645735706549494b64573570654842685932746c64494948596e566d59323975626f6345667741414159635141414141414141410a41414141414141414141414141596345724273414254414b42676771686b6a4f5051514441674e4941444246416945413873616c4a667134476671465557532f0a35347a335461746c6447736673796a4a383035425a5263334f326f434943794e6e3975716976566f5575365935345143624c3966394c575779547a516e61616e0a656977482f51696b0a2d2d2d2d2d454e442043455254494649434154452d2d2d2d2d0a
53+
LND_MACAROON=0201036c6e6402f801030a106cf4e146abffa5d766befbbf4c73b5a31201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e6572617465120472656164000006202c3bfd55c191e925cbffd73712c9d4b9b4a8440410bde5f8a0a6e33af8b3d876
54+
LND_SOCKET=sn_lnd:10009
55+
56+
# nostr (NIP-57 zap receipts)
57+
# openssl rand -hex 32
58+
NOSTR_PRIVATE_KEY=5f30b7e7714360f51f2be2e30c1d93b7fdf67366e730658e85777dfcc4e4245f
59+
60+
# imgproxy
61+
NEXT_PUBLIC_IMGPROXY_URL=http://localhost:3001
62+
IMGPROXY_KEY=9c273e803fd5d444bf8883f8c3000de57bee7995222370cab7f2d218dd9a4bbff6ca11cbf902e61eeef4358616f231da51e183aee6841e3a797a5c9a9530ba67
63+
IMGPROXY_SALT=47b802be2c9250a66b998f411fc63912ab0bc1c6b47d99b8d37c61019d1312a984b98745eac83db9791b01bb8c93ecbc9b2ef9f2981d66061c7d0a4528ff6465
64+
7665
IMGPROXY_ENABLE_WEBP_DETECTION=1
7766
IMGPROXY_ENABLE_AVIF_DETECTION=1
7867
IMGPROXY_MAX_ANIMATION_FRAMES=2000
@@ -84,11 +73,67 @@ IMGPROXY_DOWNLOAD_TIMEOUT=9
8473
# IMGPROXY_DEVELOPMENT_ERRORS_MODE=1
8574
# IMGPROXY_ENABLE_DEBUG_HEADERS=true
8675

76+
NEXT_PUBLIC_AWS_UPLOAD_BUCKET=uploads
77+
NEXT_PUBLIC_MEDIA_DOMAIN=localhost:4566
78+
NEXT_PUBLIC_MEDIA_URL=http://localhost:4566/uploads
79+
80+
# search
81+
OPENSEARCH_URL=http://opensearch:9200
82+
OPENSEARCH_USERNAME=
83+
OPENSEARCH_PASSWORD=
84+
OPENSEARCH_INDEX=item
85+
OPENSEARCH_MODEL_ID=
86+
8787
# prisma db url
8888
DATABASE_URL="postgresql://sn:password@db:5432/stackernews?schema=public"
8989

90+
###################
91+
# FOR DOCKER ONLY #
92+
###################
93+
94+
# containers can't use localhost, so we need to use the container name
95+
IMGPROXY_URL_DOCKER=http://imgproxy:8080
96+
MEDIA_URL_DOCKER=http://s3:4566/uploads
97+
9098
# postgres container stuff
9199
POSTGRES_PASSWORD=password
92100
POSTGRES_USER=sn
93101
POSTGRES_DB=stackernews
94102

103+
# opensearch container stuff
104+
OPENSEARCH_INITIAL_ADMIN_PASSWORD=mVchg1T5oA9wudUh
105+
plugins.security.disabled=true
106+
discovery.type=single-node
107+
DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
108+
109+
# bitcoind container stuff
110+
RPC_AUTH='7c68e5fcdba94a366bfdf629ecc676bb$0d0fc087c3bf7f068f350292bf8de1418df3dd8cb31e35682d5d3108d601002b'
111+
RPC_USER=bitcoin
112+
RPC_PASS=bitcoin
113+
RPC_PORT=18443
114+
P2P_PORT=18444
115+
ZMQ_BLOCK_PORT=28334
116+
ZMQ_TX_PORT=28335
117+
118+
# sn lnd container stuff
119+
LND_REST_PORT=8080
120+
LND_GRPC_PORT=10009
121+
LND_P2P_PORT=9735
122+
# docker exec -u lnd sn_lnd lncli newaddress p2wkh --unused
123+
LND_ADDR=bcrt1q7q06n5st4vqq3lssn0rtkrn2qqypghv9xg2xnl
124+
LND_PUBKEY=02cb2e2d5a6c5b17fa67b1a883e2973c82e328fb9bd08b2b156a9e23820c87a490
125+
126+
# stacker lnd container stuff
127+
STACKER_LND_REST_PORT=8081
128+
STACKER_LND_GRPC_PORT=10010
129+
# docker exec -u lnd stacker_lnd lncli newaddress p2wkh --unused
130+
STACKER_LND_ADDR=bcrt1qfqau4ug9e6rtrvxrgclg58e0r93wshucumm9vu
131+
STACKER_LND_PUBKEY=028093ae52e011d45b3e67f2e0f2cb6c3a1d7f88d2920d408f3ac6db3a56dc4b35
132+
133+
LNCLI_NETWORK=regtest
134+
135+
# localstack container stuff
136+
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
137+
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
138+
PERSISTENCE=1
139+
SKIP_SSL_CERT_DOWNLOAD=1

‎.gitignore‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,9 @@ yarn-debug.log*
2828
yarn-error.log*
2929

3030
# local env files
31-
.env
3231
envbak
33-
.env.local
34-
.env.development.local
35-
.env.test.local
36-
.env.production.local
32+
.env*
33+
!.env.sample
3734

3835
# vercel
3936
.vercel

‎Dockerfile‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,16 @@ FROM node:18.17.0-bullseye
44

55
ENV NODE_ENV=development
66

7+
ARG UID
8+
ARG GID
9+
RUN groupadd -fg "$GID" apprunner
10+
RUN useradd -om -u "$UID" -g "$GID" apprunner
11+
USER apprunner
12+
713
WORKDIR /app
814

915
EXPOSE 3000
1016

11-
CMD npm install --loglevel verbose --legacy-peer-deps; npx prisma migrate dev; npm run dev
17+
COPY package.json package-lock.json ./
18+
RUN npm ci --legacy-peer-deps --loglevel verbose
19+
CMD ["sh","-c","npm install --loglevel verbose --legacy-peer-deps && npx prisma migrate dev && npm run dev"]

‎api/s3/index.js‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import AWS from 'aws-sdk'
2+
import { MEDIA_URL } from '../../lib/constants'
23

34
const bucketRegion = 'us-east-1'
45
const Bucket = process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET
@@ -7,8 +8,18 @@ AWS.config.update({
78
region: bucketRegion
89
})
910

11+
const config = {
12+
apiVersion: '2006-03-01',
13+
endpoint: process.env.NODE_ENV === 'development' ? `${MEDIA_URL}` : undefined,
14+
s3ForcePathStyle: process.env.NODE_ENV === 'development'
15+
}
16+
1017
export function createPresignedPost ({ key, type, size }) {
11-
const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
18+
const s3 = new AWS.S3({
19+
...config,
20+
// in development, we need to be able to call this from localhost
21+
endpoint: process.env.NODE_ENV === 'development' ? `${process.env.NEXT_PUBLIC_MEDIA_URL}` : undefined
22+
})
1223
return new Promise((resolve, reject) => {
1324
s3.createPresignedPost({
1425
Bucket,
@@ -25,7 +36,7 @@ export function createPresignedPost ({ key, type, size }) {
2536
}
2637

2738
export async function deleteObjects (keys) {
28-
const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
39+
const s3 = new AWS.S3(config)
2940
// max 1000 keys per request
3041
// see https://docs.aws.amazon.com/cli/latest/reference/s3api/delete-objects.html
3142
const batchSize = 1000

‎components/image.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { IMGPROXY_URL_REGEXP } from '../lib/url'
44
import { useShowModal } from './modal'
55
import { useMe } from './me'
66
import { Dropdown } from 'react-bootstrap'
7-
import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW } from '../lib/constants'
7+
import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW, MEDIA_URL } from '../lib/constants'
88
import { useToast } from './toast'
99
import gql from 'graphql-tag'
1010
import { useMutation } from '@apollo/client'
@@ -68,6 +68,10 @@ function ImageProxy ({ src, srcSet: { dimensions, ...srcSetObj } = {}, onClick,
6868
if (!srcSetObj) return undefined
6969
// srcSetObj shape: { [widthDescriptor]: <imgproxyUrl>, ... }
7070
return Object.entries(srcSetObj).reduce((acc, [wDescriptor, url], i, arr) => {
71+
// backwards compatibility: we used to replace image urls with imgproxy urls rather just storing paths
72+
if (!url.startsWith('http')) {
73+
url = `${process.env.NEXT_PUBLIC_IMGPROXY_URL}${url}`
74+
}
7175
return acc + `${url} ${wDescriptor}` + (i < arr.length - 1 ? ', ' : '')
7276
}, '')
7377
}, [srcSetObj])
@@ -77,6 +81,9 @@ function ImageProxy ({ src, srcSet: { dimensions, ...srcSetObj } = {}, onClick,
7781
const bestResSrc = useMemo(() => {
7882
if (!srcSetObj) return src
7983
return Object.entries(srcSetObj).reduce((acc, [wDescriptor, url]) => {
84+
if (!url.startsWith('http')) {
85+
url = `${process.env.NEXT_PUBLIC_IMGPROXY_URL}${url}`
86+
}
8087
const w = Number(wDescriptor.replace(/w$/, ''))
8188
return w > acc.w ? { w, url } : acc
8289
}, { w: 0, url: undefined }).url
@@ -224,7 +231,7 @@ export const ImageUpload = forwardRef(({ children, className, onSelect, onUpload
224231
return
225232
}
226233

227-
const url = `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${data.getSignedPOST.fields.key}`
234+
const url = `${MEDIA_URL}/${data.getSignedPOST.fields.key}`
228235
// key is upload id in database
229236
const id = data.getSignedPOST.fields.key
230237
onSuccess?.({ ...variables, id, name: file.name, url, file })

‎components/item-job.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { timeSince } from '../lib/time'
1010
import EmailIcon from '../svgs/mail-open-line.svg'
1111
import Share from './share'
1212
import Hat from './hat'
13+
import { MEDIA_URL } from '../lib/constants'
1314

1415
export default function ItemJob ({ item, toc, rank, children }) {
1516
const isEmail = string().email().isValidSync(item.url)
@@ -25,7 +26,7 @@ export default function ItemJob ({ item, toc, rank, children }) {
2526
<div className={styles.item}>
2627
<Link href={`/items/${item.id}`}>
2728
<Image
28-
src={item.uploadId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
29+
src={item.uploadId ? `${MEDIA_URL}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
2930
/>
3031
</Link>
3132
<div className={`${styles.hunk} align-self-center mb-0`}>

‎components/job-form.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import Link from 'next/link'
1515
import { usePrice } from './price'
1616
import Avatar from './avatar'
1717
import { jobSchema } from '../lib/validate'
18-
import { MAX_TITLE_LENGTH } from '../lib/constants'
18+
import { MAX_TITLE_LENGTH, MEDIA_URL } from '../lib/constants'
1919
import { useToast } from './toast'
2020
import { toastDeleteScheduled } from '../lib/form'
2121
import { ItemButtonBar } from './post'
@@ -110,7 +110,7 @@ export default function JobForm ({ item, sub }) {
110110
<label className='form-label'>logo</label>
111111
<div className='position-relative' style={{ width: 'fit-content' }}>
112112
<Image
113-
src={logoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
113+
src={logoId ? `${MEDIA_URL}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
114114
/>
115115
<Avatar onSuccess={setLogoId} />
116116
</div>

‎components/user-header.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import { hexToBech32 } from '../lib/nostr'
2828
import NostrIcon from '../svgs/nostr.svg'
2929
import GithubIcon from '../svgs/github-fill.svg'
3030
import TwitterIcon from '../svgs/twitter-fill.svg'
31-
import { UNKNOWN_LINK_REL } from '../lib/constants'
31+
import { UNKNOWN_LINK_REL, MEDIA_URL } from '../lib/constants'
3232

3333
export default function UserHeader ({ user }) {
3434
const router = useRouter()
@@ -96,7 +96,7 @@ function HeaderPhoto ({ user, isMe }) {
9696
}
9797
}
9898
)
99-
const src = user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'
99+
const src = user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'
100100

101101
return (
102102
<div className='position-relative align-self-start' style={{ width: 'fit-content' }}>

‎components/user-list.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { useQuery } from '@apollo/client'
88
import MoreFooter from './more-footer'
99
import { useData } from './use-data'
1010
import Hat from './hat'
11+
import { MEDIA_URL } from '../lib/constants'
1112

1213
// all of this nonsense is to show the stat we are sorting by first
1314
const Stacked = ({ user }) => (user.optional.stacked !== null && <span>{abbrNum(user.optional.stacked)} stacked</span>)
@@ -48,7 +49,7 @@ function User ({ user, rank, statComps, Embellish }) {
4849
<div className={`${styles.item} mb-2`}>
4950
<Link href={`/${user.name}`}>
5051
<Image
51-
src={user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
52+
src={user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
5253
className={`${userStyles.userimg} me-2`}
5354
/>
5455
</Link>

0 commit comments

Comments
 (0)